feat: distinguish smoke skips from passes and assert post-exit sidecars #129
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Blender Smoke Test | ||
| # Executes the snippets' and skills' headline examples inside REAL Blender, headless, | ||
| # on the current stable (5.2.x LTS) and the fallback LTS (4.5.x) on every PR, | ||
| # plus 5.1.x on the weekly cron, and fails on any error or | ||
| # empty-output assertion. Shipped examples go through tests/smoke/run_example.py | ||
| # (catalog.json): SKIP is exit 77 + SMOKE_SKIP below --min-version, never exit 0. | ||
| # Post-exit sidecars are opt-in. A leg with zero PASSes is red. | ||
| # py_compile (in validate.yml) cannot catch API-level regressions | ||
| # like the EEVEE-id inversion, the slotted-actions boundary, the driver TypeError, or the | ||
| # dead SDF link -- this gate runs the code so those surface in CI, not in users' files. | ||
| on: | ||
| workflow_dispatch: {} | ||
| schedule: | ||
| - cron: "0 7 * * 1" # weekly, Monday 07:00 UTC | ||
| pull_request: | ||
| branches: [main] | ||
| permissions: | ||
| contents: read | ||
| concurrency: | ||
| group: blender-smoke-${{ github.ref }} | ||
| cancel-in-progress: true | ||
| jobs: | ||
| smoke: | ||
| name: Blender ${{ matrix.series }} smoke | ||
| runs-on: ubuntu-latest | ||
| timeout-minutes: 45 | ||
| env: | ||
| BDT_SMOKE_STATUS: ${{ runner.temp }}/smoke-status.jsonl | ||
| strategy: | ||
| fail-fast: false | ||
| matrix: | ||
| series: ${{ github.event_name == 'schedule' && fromJSON('["5.2","5.1","4.5"]') || fromJSON('["5.2","4.5"]') }} | ||
| steps: | ||
| - uses: actions/checkout@v7 | ||
| - name: Harness protocol unit tests | ||
| run: | | ||
| set -euo pipefail | ||
| python3 tests/smoke/test_harness.py -v | ||
| - name: Install Blender runtime libraries | ||
| run: | | ||
| set -euo pipefail | ||
| sudo apt-get update | ||
| # Blender needs these shared libs even in --background (GPU/GL module init); | ||
| # xvfb provides a virtual display so GL/EGL init does not abort the process. | ||
| sudo apt-get install -y --no-install-recommends \ | ||
| xvfb libgl1 libegl1 libxrender1 libxxf86vm1 libxfixes3 libxi6 \ | ||
| libxkbcommon0 libsm6 libice6 | ||
| - name: Resolve and download Blender ${{ matrix.series }} | ||
| run: | | ||
| set -euo pipefail | ||
| series="${{ matrix.series }}" | ||
| base="https://download.blender.org/release/Blender${series}/" | ||
| echo "Listing $base" | ||
| # pick the highest point release for this series (linux x64 portable) | ||
| file=$(curl -fsSL "$base" \ | ||
| | grep -oE "blender-${series}\.[0-9]+-linux-x64\.tar\.xz" \ | ||
| | sort -V | uniq | tail -1) | ||
| if [ -z "$file" ]; then | ||
| echo "::error::Could not resolve a linux-x64 build for Blender ${series} at $base" | ||
| exit 1 | ||
| fi | ||
| url="${base}${file}" | ||
| echo "Downloading $url" | ||
| mkdir -p "$RUNNER_TEMP/bl" | ||
| curl -fSL --retry 3 -o "$RUNNER_TEMP/bl.tar.xz" "$url" | ||
| tar -xf "$RUNNER_TEMP/bl.tar.xz" -C "$RUNNER_TEMP/bl" | ||
| bl=$(find "$RUNNER_TEMP/bl" -maxdepth 2 -type f -name blender | head -1) | ||
| if [ -z "$bl" ]; then | ||
| echo "::error::blender binary not found after extraction" | ||
| exit 1 | ||
| fi | ||
| echo "BLENDER=$bl" >> "$GITHUB_ENV" | ||
| - name: Print Blender version | ||
| run: | | ||
| set -euo pipefail | ||
| "$BLENDER" --version | head -1 | ||
| # series guard: confirm we actually got the matrix series | ||
| "$BLENDER" --version | head -1 | grep -q "Blender ${{ matrix.series }}\." \ | ||
| || { echo "::error::version does not match series ${{ matrix.series }}"; exit 1; } | ||
| - name: Run in-Blender smoke driver | ||
| run: | | ||
| set -euo pipefail | ||
| mkdir -p "$RUNNER_TEMP/out" | ||
| xvfb-run -a "$BLENDER" --background --python tests/smoke/run_smoke.py -- "$RUNNER_TEMP/out" | ||
| - name: Build template input scene | ||
| run: | | ||
| set -euo pipefail | ||
| xvfb-run -a "$BLENDER" --background --python tests/smoke/make_input.py -- "$RUNNER_TEMP/out/input.blend" | ||
| - name: Headless glTF template runs (exit 0, .glb produced) | ||
| run: | | ||
| set -euo pipefail | ||
| xvfb-run -a "$BLENDER" --background "$RUNNER_TEMP/out/input.blend" \ | ||
| --python tests/smoke/tmpl_gltf.py -- \ | ||
| --output "$RUNNER_TEMP/out/out.glb" --apply-modifier SUBSURF | ||
| test -s "$RUNNER_TEMP/out/out.glb" || { echo "::error::glTF output missing/empty"; exit 1; } | ||
| head -c4 "$RUNNER_TEMP/out/out.glb" | grep -q "glTF" || { echo "::error::not a glTF binary"; exit 1; } | ||
| - name: Headless template no-mesh path returns exit 2 | ||
| run: | | ||
| set +e | ||
| xvfb-run -a "$BLENDER" --background "$RUNNER_TEMP/out/empty.blend" \ | ||
| --python tests/smoke/tmpl_gltf.py -- --output "$RUNNER_TEMP/out/none.glb" | ||
| code=$? | ||
| set -e | ||
| [ "$code" -eq 2 ] || { echo "::error::expected exit 2 for no-mesh input, got $code"; exit 1; } | ||
| echo "no-mesh exit code = $code (correct)" | ||
| - name: Headless render template runs (exit 0, PNG produced) | ||
| run: | | ||
| set -euo pipefail | ||
| # Cycles (CPU) so this is reliable on GPU-less runners; the EEVEE-id regression | ||
| # itself is gated in run_smoke.py via engine assignment. | ||
| xvfb-run -a "$BLENDER" --background "$RUNNER_TEMP/out/input.blend" \ | ||
| --python tests/smoke/tmpl_render.py -- \ | ||
| --output "$RUNNER_TEMP/out/render.png" --engine CYCLES | ||
| test -s "$RUNNER_TEMP/out/render.png" || { echo "::error::render PNG missing/empty"; exit 1; } | ||
| - name: Canary - expected skip is SKIP not PASS | ||
| run: | | ||
| set -euo pipefail | ||
| python3 tests/smoke/run_example.py \ | ||
| --name canary-skip \ | ||
| --blender "$BLENDER" \ | ||
| --script tests/smoke/canary_skip.py \ | ||
| --series "${{ matrix.series }}" \ | ||
| --min-version 99.0 \ | ||
| --xvfb | ||
| - name: Canary - unexpected skip is red | ||
| run: | | ||
| set +e | ||
| python3 tests/smoke/run_example.py --no-record \ | ||
| --name canary-skip-unexpected \ | ||
| --blender "$BLENDER" \ | ||
| --script tests/smoke/canary_skip.py \ | ||
| --series "${{ matrix.series }}" \ | ||
| --min-version 4.0 \ | ||
| --xvfb | ||
| code=$? | ||
| set -e | ||
| [ "$code" -eq 1 ] || { echo "::error::unexpected skip did not fail (got $code)"; exit 1; } | ||
| echo "unexpected skip exited 1 (correct)" | ||
| - name: Canary - post-exit sidecar | ||
| run: | | ||
| set -euo pipefail | ||
| python3 tests/smoke/run_example.py \ | ||
| --name canary-sidecar \ | ||
| --blender "$BLENDER" \ | ||
| --script tests/smoke/canary_sidecar.py \ | ||
| --series "${{ matrix.series }}" \ | ||
| --expect-sidecar "$RUNNER_TEMP/out/canary.sidecar" \ | ||
| --sidecar-contains sidecar-ok \ | ||
| --xvfb | ||
| - name: Canary - missing sidecar is red | ||
| run: | | ||
| set +e | ||
| python3 tests/smoke/run_example.py --no-record \ | ||
| --name canary-sidecar-missing \ | ||
| --blender "$BLENDER" \ | ||
| --script tests/smoke/canary_sidecar.py \ | ||
| --series "${{ matrix.series }}" \ | ||
| --expect-sidecar "$RUNNER_TEMP/out/missing.sidecar" \ | ||
| --sidecar-contains sidecar-ok \ | ||
| --xvfb \ | ||
| -- --omit-sidecar | ||
| code=$? | ||
| set -e | ||
| [ "$code" -eq 1 ] || { echo "::error::missing sidecar did not fail (got $code)"; exit 1; } | ||
| echo "missing sidecar exited 1 (correct)" | ||
| - name: Shipped examples | ||
| run: | | ||
| set -euo pipefail | ||
| python3 tests/smoke/run_catalog.py \ | ||
| --blender "$BLENDER" \ | ||
| --series "${{ matrix.series }}" \ | ||
| --status "$BDT_SMOKE_STATUS" \ | ||
| --out "$RUNNER_TEMP/out" \ | ||
| --xvfb \ | ||
| --catalog tests/smoke/catalog.json | ||
| - name: Smoke summary | ||
| if: ${{ always() && env.BLENDER != '' }} | ||
| run: | | ||
| set -euo pipefail | ||
| python3 tests/smoke/summarize.py --status "$BDT_SMOKE_STATUS" | ||
