diff --git a/.github/workflows/_release-line.yml b/.github/workflows/_release-line.yml new file mode 100644 index 0000000..8e8b2dc --- /dev/null +++ b/.github/workflows/_release-line.yml @@ -0,0 +1,152 @@ +# Per-line release unit (release-src.yml calls one per ruby line): the +# line tip's compile smoke FIRST, the line's build legs only when the tip +# is green, the line's carry-forward copies in parallel (they verify +# against the previous release, never the smoke). One line's failure +# blocks only that line. +# +# All matrices come from the shared tools (tools/build_matrix, +# tools/smoke_matrix with --line) — this workflow carries no matrix +# logic of its own. +name: release-line + +on: + workflow_call: + inputs: + line: + description: The ruby line this unit releases (3.1 / 3.2 / 3.3 / 3.4 / 4.0) + required: true + type: string + tag: + description: The tag being published + required: true + type: string + +jobs: + plan: + name: plan ${{ inputs.line }} + runs-on: ubuntu-latest + outputs: + smoke: ${{ steps.emit.outputs.smoke }} + build: ${{ steps.emit.outputs.build }} + copies: ${{ steps.emit.outputs.copies }} + previous_tag: ${{ steps.emit.outputs.previous_tag }} + smoke_count: ${{ steps.emit.outputs.smoke_count }} + build_count: ${{ steps.emit.outputs.build_count }} + copies_count: ${{ steps.emit.outputs.copies_count }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + - name: fetch tags (previous-release diff base) + run: git fetch --tags --force + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + ruby-version: "3.2" + - id: emit + run: | + smoke=$(tools/smoke_matrix "${{ inputs.tag }}" --line "${{ inputs.line }}") + build=$(tools/build_matrix "${{ inputs.tag }}" --build --line "${{ inputs.line }}") + copies=$(tools/build_matrix "${{ inputs.tag }}" --copies --line "${{ inputs.line }}") + { + echo "smoke=$smoke" + echo "build=$build" + echo "copies=$copies" + echo "previous_tag=$(tools/build_matrix "${{ inputs.tag }}" --previous-tag)" + # Leg counts drive the job-level if: guards (an unguarded + # empty matrix materializes no legs and poisons the run + # conclusion — the v0.2.15 lesson, release-src.yml history). + echo "smoke_count=$(jq '.include | length' <<< "$smoke")" + echo "build_count=$(jq '.include | length' <<< "$build")" + echo "copies_count=$(jq '.include | length' <<< "$copies")" + } >> "$GITHUB_OUTPUT" + + # The line tip FIRST: configure the tree of the line's newest version + # per affected scenario and compile each patched translation unit + # (objects only — no runtime build, no link, no exts) against the + # vendored stub tebako headers in ci/include. Only the scenarios the + # changed patches FEED get a leg (tools/smoke_matrix). + smoke: + name: smoke ${{ inputs.line }} (${{ matrix.version }} / ${{ matrix.platform }}) + needs: plan + if: needs.plan.outputs.smoke_count != '0' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.smoke) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + ruby-version: "3.2" + - name: scenario toolchain + run: | + sudo apt-get update -qq + case "${{ matrix.platform }}" in + linux-musl) sudo apt-get install -y -qq musl-tools ;; + msys) sudo apt-get install -y -qq gcc-mingw-w64-x86-64 ;; + esac + - name: configure + compile patched translation units + run: tools/compile_smoke "${{ matrix.version }}" "$PWD/build/compile-smoke" --platform "${{ matrix.platform }}" + + # The line's build legs run ONLY when the tip smoke passed (or was + # vacuous — nothing about the line's patches changed). A failed smoke + # blocks this line's builds and nothing else. + build: + name: build ${{ inputs.line }} (${{ matrix.version }}${{ matrix.suffix }}) + needs: [plan, smoke] + if: ${{ needs.plan.outputs.build_count != '0' && (needs.smoke.result == 'success' || needs.smoke.result == 'skipped') }} + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.build) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + ruby-version: "3.2" + bundler-cache: true + - name: validate manifests against schema + run: bundle exec tools/validate_manifests + - name: apply patches + run: tools/apply "${{ matrix.version }}" "$PWD/build" --platform "${{ matrix.platform }}" --pass "${{ matrix.pass }}" + - name: package + working-directory: build + run: | + tar -czf "${{ matrix.asset }}" "${{ matrix.tree }}" + sha256sum "${{ matrix.asset }}" > "${{ matrix.asset }}.sha256" + - name: extract-verify + working-directory: build + run: | + mkdir verify + tar -xzf "${{ matrix.asset }}" -C verify + diff -qr "${{ matrix.tree }}" "verify/${{ matrix.tree }}" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tfs-ruby-${{ matrix.version }}-src${{ matrix.suffix }} + path: build/${{ matrix.asset }}* + retention-days: 1 + + # The line's carry-forward copies: sha256-verified against the previous + # release's published SHA256SUMS (tools/copy_asset). Independent of the + # smoke — a copied asset is byte-identical with what the previous + # release published. + copy: + name: copy ${{ inputs.line }} (${{ matrix.asset }}) + needs: plan + if: needs.plan.outputs.copies_count != '0' + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.plan.outputs.copies) }} + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 + with: + ruby-version: "3.2" + - name: download + verify against the previous release's SHA256SUMS + run: tools/copy_asset "${{ needs.plan.outputs.previous_tag }}" "${{ matrix.asset }}" "$PWD/dist" + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: tfs-ruby-${{ matrix.version }}-src${{ matrix.suffix }} + path: dist/${{ matrix.asset }}* + retention-days: 1 diff --git a/.github/workflows/release-src.yml b/.github/workflows/release-src.yml index fd62be5..5a3eb36 100644 --- a/.github/workflows/release-src.yml +++ b/.github/workflows/release-src.yml @@ -11,197 +11,58 @@ on: default: v0.0.0 jobs: - versions: - name: plan build + copy matrices (diff-aware) + plan: + name: plan the per-line release units (diff-aware) runs-on: ubuntu-latest outputs: - matrix: ${{ steps.emit.outputs.matrix }} - copies: ${{ steps.emit.outputs.copies }} - build_count: ${{ steps.emit.outputs.build_count }} - copies_count: ${{ steps.emit.outputs.copies_count }} - previous_tag: ${{ steps.emit.outputs.previous_tag }} + lines: ${{ steps.emit.outputs.lines }} + lines_count: ${{ steps.emit.outputs.lines_count }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - name: fetch tags (previous-release diff base) - run: git fetch --tags --force - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 with: ruby-version: "3.2" - # Fault isolation (tools/build_matrix over Tfs::ReleaseDiff, the same - # previous-tag diff base the compile-smoke gate uses): only versions - # of CHANGED patch lines compile (plus every version on a shared - # tooling change, and versions whose versions.yml entry is new or - # moved); every unchanged version is carried forward from the - # previous release as a sha256-verified copy (the copy job below). + # One release unit per line PRESENT in versions.yml: changed lines + # build (their smoke gates their builds), unchanged lines carry + # their assets forward as sha256-verified copies. The line's own + # unit recomputes its rows from the shared tools (tools/build_matrix + # --line / tools/smoke_matrix --line over Tfs::ReleaseDiff's + # patch→scenario attribution) — no matrix logic lives in YAML. - id: emit env: RELEASE_TAG: ${{ github.event_name == 'push' && github.ref_name || inputs.release_tag }} run: | - build=$(tools/build_matrix "$RELEASE_TAG" --build) - copies=$(tools/build_matrix "$RELEASE_TAG" --copies) - { - echo "matrix=$build" - echo "copies=$copies" - # Leg counts drive the job-level if: guards on build/copy: an - # UNGUARDED empty matrix ({"include":[]}) materializes no legs - # and no check run, yet the orchestrator reports the job as - # not-success/not-skipped and fails the whole run (the v0.2.15 - # tag runs 30975733360/30978817396: 165/165 legs green, publish - # skipped, run failure, because the copies plan was empty). - # Same count-guard pattern as release-monitor.yml's onboard job. - echo "build_count=$(jq '.include | length' <<< "$build")" - echo "copies_count=$(jq '.include | length' <<< "$copies")" - echo "previous_tag=$(tools/build_matrix "$RELEASE_TAG" --previous-tag)" - } >> "$GITHUB_OUTPUT" - - changes: - name: map changed patch lines to compile-smoke legs - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.emit.outputs.matrix }} - smoke_count: ${{ steps.emit.outputs.smoke_count }} - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - - name: fetch tags (previous-release diff base) - run: git fetch --tags --force - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 - with: - ruby-version: "3.2" - - id: emit - run: | - matrix=$(tools/smoke_matrix "${{ github.event_name == 'push' && github.ref_name || inputs.release_tag }}") + lines=$(tools/build_matrix "$RELEASE_TAG" --lines) { - echo "matrix=$matrix" - # Leg count drives the job-level if: guard on compile-smoke - # (why an unguarded empty matrix breaks the run: see the - # versions job's emit step). - echo "smoke_count=$(jq '.include | length' <<< "$matrix")" + echo "lines=$lines" + # The count drives the job-level if: on release (an + # unguarded empty matrix poisons the run conclusion). + echo "lines_count=$(jq 'length' <<< "$lines")" } >> "$GITHUB_OUTPUT" - # Publish gate (roadmap 17.0; the v0.2.8 lesson: a patch release shipped - # apply-clean but uncompilable and broke every linux runtime leg). For - # every line whose patch set changed in the tag (tools/smoke_matrix diffs - # the tag against the previous release tag), configure the tree and - # compile each patched translation unit of the line's NEWEST version per - # shipped scenario -- a representative per-line leg, not per-version: - # patches are line-wide, so per-version legs would multiply configure - # runs for no extra signal. The compile is the cheapest that catches a - # broken shim: ./configure + make per patched .c (objects only -- - # no runtime build, no link, no exts), against the vendored stub tebako - # headers in ci/include. msys legs compile at pass 2 (the pass split - # selects GNUmakefile variants; no patched .c differs between passes). - # With no changed patch set the plan is empty and the gate is vacuous: - # the job-level if: below then skips the job CLEANLY (an unguarded empty - # matrix would poison the run conclusion -- see the versions job's emit - # step), and publish's if: treats that skipped as pass, failure as block. - compile-smoke: - name: compile-smoke ${{ matrix.version }} (${{ matrix.platform }}) - needs: changes - if: needs.changes.outputs.smoke_count != '0' - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.changes.outputs.matrix) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 - with: - ruby-version: "3.2" - - name: scenario toolchain - run: | - sudo apt-get update -qq - case "${{ matrix.platform }}" in - linux-musl) sudo apt-get install -y -qq musl-tools ;; - msys) sudo apt-get install -y -qq gcc-mingw-w64-x86-64 ;; - esac - - name: configure + compile patched translation units - run: tools/compile_smoke "${{ matrix.version }}" "$PWD/build/compile-smoke" --platform "${{ matrix.platform }}" - - build: - name: tfs-ruby-${{ matrix.version }}-src${{ matrix.suffix }} - needs: versions - if: needs.versions.outputs.build_count != '0' + # The per-line release units: each line's tip smoke runs FIRST and + # gates only its own line's builds; copies run independently. A line + # that fails blocks the publish (its changed assets must never be + # back-filled with stale copies) but never another line's work. + release: + name: release ${{ matrix.line }} + needs: plan + if: needs.plan.outputs.lines_count != '0' strategy: fail-fast: false - matrix: ${{ fromJSON(needs.versions.outputs.matrix) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 - with: - ruby-version: "3.2" - bundler-cache: true - - name: validate manifests against schema - run: bundle exec tools/validate_manifests - - name: apply patches - run: tools/apply "${{ matrix.version }}" "$PWD/build" --platform "${{ matrix.platform }}" --pass "${{ matrix.pass }}" - - name: package - working-directory: build - run: | - tar -czf "${{ matrix.asset }}" "${{ matrix.tree }}" - sha256sum "${{ matrix.asset }}" > "${{ matrix.asset }}.sha256" - - name: extract-verify - working-directory: build - run: | - mkdir verify - tar -xzf "${{ matrix.asset }}" -C verify - diff -qr "${{ matrix.tree }}" "verify/${{ matrix.tree }}" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: tfs-ruby-${{ matrix.version }}-src${{ matrix.suffix }} - path: build/${{ matrix.asset }}* - retention-days: 1 - - # Versions whose inputs did not move since the previous release are NOT - # rebuilt: the previous release's tarballs are carried forward - # sha256-verified against the previous release's published SHA256SUMS - # (tools/copy_asset -- a failed verification fails the leg loudly, naming - # the asset). The new release's asset set stays complete (changed builds - # + verified copies), byte-identical with what the previous release - # published. An empty copies plan skips the job via the same count guard - # as compile-smoke (a vacuous pass for publish's if:). - copy: - name: copy tfs-ruby-${{ matrix.version }}-src${{ matrix.suffix }} (verified from ${{ needs.versions.outputs.previous_tag }}) - needs: versions - if: needs.versions.outputs.copies_count != '0' - strategy: - fail-fast: false - matrix: ${{ fromJSON(needs.versions.outputs.copies) }} - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - - uses: ruby/setup-ruby@95ef2b042f9d7a56d8268cba8559e2842e2ad01b # v1.321.0 - with: - ruby-version: "3.2" - - name: download + verify against the previous release's SHA256SUMS - run: tools/copy_asset "${{ needs.versions.outputs.previous_tag }}" "${{ matrix.asset }}" "$PWD/dist" - - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: tfs-ruby-${{ matrix.version }}-src${{ matrix.suffix }} - path: dist/${{ matrix.asset }}* - retention-days: 1 + matrix: + line: ${{ fromJSON(needs.plan.outputs.lines) }} + uses: ./.github/workflows/_release-line.yml + with: + line: ${{ matrix.line }} + tag: ${{ github.event_name == 'push' && github.ref_name || inputs.release_tag }} publish: name: publish release - needs: [versions, build, copy, compile-smoke, changes] - # build, copy and compile-smoke run legs only where the diff-aware plan - # says so; each carries a job-level if: guard on its leg count, so an - # empty plan (nothing to build / nothing to copy / no changed patch - # set) skips the job CLEANLY (result skipped) instead of expanding an - # empty matrix -- an unguarded empty matrix materializes no legs and no - # check run, its needs..result is neither 'success' nor 'skipped', - # and the orchestrator fails the whole run (v0.2.15: runs 30975733360 - # and 30978817396 had 165/165 legs green yet publish skipped and the - # runs concluded failure, because the copies plan was empty). Publish - # only when the plan was computed (versions + changes success), every - # needed leg passed, and no leg failed or was cancelled. A skipped leg - # behind a FAILED plan job is ruled out by the two strict success - # requirements. - if: ${{ always() && needs.versions.result == 'success' && needs.changes.result == 'success' && (needs.build.result == 'success' || needs.build.result == 'skipped') && (needs.copy.result == 'success' || needs.copy.result == 'skipped') && (needs.compile-smoke.result == 'success' || needs.compile-smoke.result == 'skipped') }} + needs: [plan, release] + # Every line unit succeeded (or there was nothing to do): a failed + # line blocks the publish rather than shipping its stale copies. + if: ${{ always() && needs.plan.result == 'success' && (needs.release.result == 'success' || needs.release.result == 'skipped') }} runs-on: ubuntu-latest permissions: contents: write @@ -211,12 +72,9 @@ jobs: # next publish-gate flake must be self-explaining in the job log. - name: needs results (publish gate diagnostic) run: | - echo "versions.result = ${{ needs.versions.result }}" - echo "changes.result = ${{ needs.changes.result }}" - echo "build.result = ${{ needs.build.result }}" - echo "copy.result = ${{ needs.copy.result }}" - echo "compile-smoke.result = ${{ needs['compile-smoke'].result }}" - echo "needs (full) = ${{ toJSON(needs) }}" + echo "plan.result = ${{ needs.plan.result }}" + echo "release.result = ${{ needs.release.result }}" + echo "needs (full) = ${{ toJSON(needs) }}" - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: pattern: tfs-ruby-*-src* diff --git a/spec/tfs/build_plan_spec.rb b/spec/tfs/build_plan_spec.rb index 07c8954..659f57d 100644 --- a/spec/tfs/build_plan_spec.rb +++ b/spec/tfs/build_plan_spec.rb @@ -113,4 +113,53 @@ def previous_yaml(sha_341: "#{"0" * 63}3", with_999: true) expect(plan.builds).to eq([]) expect(plan.copies.size).to eq(7) end + + context "with an msys-only patch change" do + let(:plan) do + described_class.new(versions: versions, diff: diff_for(["patches/3.3/dir_c_memfs_msys.patch"]), previous_versions: previous) + end + + it "re-rolls only the msys tarballs of the line's versions" do + expect(plan.builds.map { |row| row[:asset] }).to eq([ + "tfs-ruby-3.3.7-src-msys-pass1.tar.gz", + "tfs-ruby-3.3.7-src-msys-pass2.tar.gz" + ]) + end + + it "carries every POSIX asset forward as a verified copy" do + expect(plan.copies.size).to eq(5) + expect(plan.copies.map { |row| row[:asset] }).to include("tfs-ruby-3.3.3-src.tar.gz", "tfs-ruby-3.3.7-src.tar.gz", "tfs-ruby-3.3.7-src-linux-musl.tar.gz") + end + end + + it "scopes a pass-marked msys patch to that pass alone" do + plan = described_class.new(versions: versions, diff: diff_for(["patches/3.3/gnumakefile_in_pass1_msys.patch"]), previous_versions: previous) + expect(plan.builds.map { |row| row[:asset] }).to eq(["tfs-ruby-3.3.7-src-msys-pass1.tar.gz"]) + expect(plan.copies.size).to eq(6) + end + + it "re-rolls only the musl tarball on a musl patch change" do + plan = described_class.new(versions: versions, diff: diff_for(["patches/3.3/thread_pthread_main_stack_musl.patch"]), previous_versions: previous) + expect(plan.builds.map { |row| row[:asset] }).to eq(["tfs-ruby-3.3.7-src-linux-musl.tar.gz"]) + expect(plan.copies.size).to eq(6) + end + + it "re-rolls nothing on a darwin-only change (no shipped scenario selects darwin today)" do + plan = described_class.new(versions: versions, diff: diff_for(["patches/3.3/configure_extstatic_bundle_loader_darwin.patch"]), previous_versions: previous) + expect(plan.builds).to eq([]) + expect(plan.copies.size).to eq(7) + end + + context "with the line filter (the per-line release workflows)" do + it "narrows builds to the line's rows" do + plan = described_class.new(versions: versions, diff: diff_for([], tags: [])) + expect(plan.builds(line: "3.3").map { |row| row[:version] }.uniq).to eq(%w[3.3.3 3.3.7]) + expect(plan.builds(line: "3.3").size).to eq(5) + end + + it "narrows copies to the line's rows" do + plan = described_class.new(versions: versions, diff: diff_for([]), previous_versions: previous) + expect(plan.copies(line: "3.4").map { |row| row[:asset] }).to eq(["tfs-ruby-3.4.1-src.tar.gz"]) + end + end end diff --git a/spec/tfs/release_diff_spec.rb b/spec/tfs/release_diff_spec.rb index 6b90cd9..83c0c09 100644 --- a/spec/tfs/release_diff_spec.rb +++ b/spec/tfs/release_diff_spec.rb @@ -89,6 +89,57 @@ def diff_with(paths) end end + describe "scenario attribution (#changed_scenarios)" do + def diff_with(paths) + described_class.new("v0.2.14", git: fake_git(tags: tags, diffs: { "v0.2.13..v0.2.14" => paths.join("\n") })) + end + + it "is nil when there is no previous tag (everything changed)" do + expect(described_class.new("v0.2.11", git: fake_git(tags: tags)).changed_scenarios).to be_nil + end + + it "attributes an msys patch to the msys scenario only" do + diff = diff_with(["patches/4.0/mkconfig_memfs_prefix_msys.patch"]) + expect(diff.changed_scenarios).to eq("4.0" => [["msys", nil]]) + end + + it "scopes a pass-marked msys patch to that pass's tarball" do + diff = diff_with(["patches/3.3/gnumakefile_in_pass1_msys.patch"]) + expect(diff.changed_scenarios).to eq("3.3" => [["msys", 1]]) + end + + it "attributes a numbered msys variant to msys" do + diff = diff_with(["patches/4.0/openssl_extconf_ruby_export_msys_6.patch"]) + expect(diff.changed_scenarios).to eq("4.0" => [["msys", nil]]) + end + + it "attributes a musl patch to linux-musl only" do + diff = diff_with(["patches/4.0/thread_pthread_main_stack_musl.patch"]) + expect(diff.changed_scenarios).to eq("4.0" => [["linux-musl", nil]]) + end + + it "attributes a darwin patch to nothing (no shipped scenario selects darwin today)" do + diff = diff_with(["patches/4.0/configure_extstatic_bundle_loader_darwin.patch"]) + expect(diff.changed_scenarios).to eq("4.0" => []) + end + + it "attributes a base patch to every scenario" do + diff = diff_with(["patches/3.4/prism_compile_memfs.patch"]) + expect(diff.changed_scenarios["3.4"]).to match_array([["linux-gnu", nil], ["linux-musl", nil], ["msys", nil]]) + end + + it "attributes a line manifest change to every scenario (fail closed)" do + diff = diff_with(["patches/3.1/patch-3.1.yaml", "patches/3.4/patch-3.4.2.yaml"]) + expect(diff.changed_scenarios["3.1"]).to match_array([["linux-gnu", nil], ["linux-musl", nil], ["msys", nil]]) + expect(diff.changed_scenarios["3.4"]).to match_array([["linux-gnu", nil], ["linux-musl", nil], ["msys", nil]]) + end + + it "unions attributions across several changed patches of one line" do + diff = diff_with(["patches/3.3/dir_c_memfs_msys.patch", "patches/3.3/io_c_tebako_includes.patch"]) + expect(diff.changed_scenarios["3.3"]).to match_array([["linux-gnu", nil], ["linux-musl", nil], ["msys", nil]]) + end + end + describe "#previous_file" do it "reads a file at the previous release tag" do git = fake_git(tags: tags, shows: { "v0.2.13:versions.yml" => "versions: {}\n" }) diff --git a/spec/tfs/smoke_plan_spec.rb b/spec/tfs/smoke_plan_spec.rb new file mode 100644 index 0000000..40ad2bc --- /dev/null +++ b/spec/tfs/smoke_plan_spec.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +RSpec.describe Tfs::SmokePlan do + subject(:versions) { Tfs::Versions.new(File.join(SPEC_FIXTURES, "versions.yml")) } + + # A real ReleaseDiff over a fake git: publishing v2, previous release v1. + def diff_for(paths, tags: %w[v2 v1]) + git = lambda do |*args| + case args[0] + when "tag" then tags.empty? ? "" : "#{tags.join("\n")}\n" + when "diff" then paths.join("\n") + else raise Tfs::ReleaseDiff::Error, "unexpected git #{args.join(' ')}" + end + end + Tfs::ReleaseDiff.new("v2", git: git) + end + + it "smokes every line's newest version per shipped scenario on the first release" do + legs = described_class.new(versions: versions, diff: diff_for([], tags: [])).legs + expect(legs).to contain_exactly( + { line: "3.3", version: "3.3.7", platform: "linux-gnu" }, + { line: "3.3", version: "3.3.7", platform: "linux-musl" }, + { line: "3.3", version: "3.3.7", platform: "msys" }, + { line: "3.4", version: "3.4.1", platform: "linux-gnu" }, + { line: "9.9", version: "9.9.9", platform: "linux-gnu" } + ) + end + + it "smokes only the msys leg on an msys-only patch change" do + legs = described_class.new(versions: versions, diff: diff_for(["patches/3.3/dir_c_memfs_msys.patch"])).legs + expect(legs).to eq([{ line: "3.3", version: "3.3.7", platform: "msys" }]) + end + + it "smokes every scenario the line ships on a base patch change" do + legs = described_class.new(versions: versions, diff: diff_for(["patches/3.3/io_c_tebako_includes.patch"])).legs + expect(legs.map { |leg| leg[:platform] }).to contain_exactly("linux-gnu", "linux-musl", "msys") + expect(legs.map { |leg| leg[:version] }.uniq).to eq(["3.3.7"]) + end + + it "smokes nothing on a darwin-only change (no shipped scenario)" do + legs = described_class.new(versions: versions, diff: diff_for(["patches/3.3/configure_extstatic_bundle_loader_darwin.patch"])).legs + expect(legs).to eq([]) + end + + it "narrows to one line with the line filter" do + diff = diff_for(["patches/3.3/io_c_tebako_includes.patch", "patches/3.4/prism_compile_memfs.patch"]) + legs = described_class.new(versions: versions, diff: diff, line: "3.4").legs + expect(legs).to eq([{ line: "3.4", version: "3.4.1", platform: "linux-gnu" }]) + end + + it "plans nothing when no patch set changed" do + legs = described_class.new(versions: versions, diff: diff_for([])).legs + expect(legs).to eq([]) + end +end diff --git a/tools/build_matrix b/tools/build_matrix index abae923..cc478c1 100755 --- a/tools/build_matrix +++ b/tools/build_matrix @@ -19,18 +19,21 @@ # --previous-tag — the bare tag the copies come from (empty on the first # release). # -# With no flag, prints all three as one JSON document. +# With no flag, prints all three as one JSON document. `--line X.Y` +# narrows the build/copy plans to one line's rows (the per-line release +# workflows each plan their own); `--lines` prints the line list itself. $LOAD_PATH.unshift(File.expand_path("lib", __dir__)) require "tfs" require "json" require "tempfile" -tag = ARGV[0] or abort "usage: tools/build_matrix [--build|--copies|--previous-tag]" +tag = ARGV[0] or abort "usage: tools/build_matrix [--build|--copies|--previous-tag|--lines] [--line X.Y]" mode = ARGV[1] || "--all" -unless %w[--build --copies --previous-tag --all].include?(mode) - abort "tools/build_matrix: unknown mode #{mode.inspect} (want --build, --copies or --previous-tag)" +unless %w[--build --copies --previous-tag --lines --all].include?(mode) + abort "tools/build_matrix: unknown mode #{mode.inspect} (want --build, --copies, --previous-tag or --lines)" end +line = ARGV[2] == "--line" ? ARGV[3] : nil diff = Tfs::ReleaseDiff.new(tag) versions = Tfs::Versions.new @@ -49,8 +52,8 @@ end plan = Tfs::BuildPlan.new(versions: versions, diff: diff, previous_versions: previous_versions) document = { - "build" => { "include" => plan.builds }, - "copies" => { "include" => plan.copies }, + "build" => { "include" => plan.builds(line: line) }, + "copies" => { "include" => plan.copies(line: line) }, "previous_tag" => diff.previous_tag } @@ -58,5 +61,6 @@ case mode when "--build" then puts JSON.generate(document["build"]) when "--copies" then puts JSON.generate(document["copies"]) when "--previous-tag" then puts diff.previous_tag +when "--lines" then puts JSON.generate(versions.map(&:line).uniq.sort) else puts JSON.generate(document) end diff --git a/tools/lib/tfs.rb b/tools/lib/tfs.rb index 9701ddf..01e580f 100644 --- a/tools/lib/tfs.rb +++ b/tools/lib/tfs.rb @@ -15,5 +15,6 @@ module Tfs autoload :Onboarder, "tfs/onboarder" autoload :ReleaseDiff, "tfs/release_diff" autoload :BuildPlan, "tfs/build_plan" + autoload :SmokePlan, "tfs/smoke_plan" autoload :ReleaseCopier, "tfs/release_copier" end diff --git a/tools/lib/tfs/build_plan.rb b/tools/lib/tfs/build_plan.rb index 5d9ed11..a8c1117 100644 --- a/tools/lib/tfs/build_plan.rb +++ b/tools/lib/tfs/build_plan.rb @@ -5,8 +5,11 @@ module Tfs # must COMPILE for the tag being published, and which are CARRIED FORWARD # from the previous release as sha256-verified copies (tools/copy_asset). # Fault isolation: a patches// change re-spends only that line's - # versions, a shared tooling change correctly re-spends everything, and a - # versions.yml change re-spends exactly the versions whose entry moved + # versions AND ONLY THE SCENARIOS THE CHANGED PATCHES FEED (an msys + # patch never re-rolls a POSIX tarball — the attribution is + # ReleaseDiff#changed_scenarios); a shared tooling change correctly + # re-spends everything, and a versions.yml change re-spends exactly + # the versions whose entry moved # (a new version has no previous asset to copy, so it always builds). # # The release's asset set stays complete either way — consumers fetch the @@ -28,36 +31,67 @@ def initialize(versions:, diff:, previous_versions: nil) # Build rows in Versions#builds shape plus the asset/tree names the # release-src legs package (the naming rule lives here, not in YAML). - def builds - rows_for(:build).map do |row| + # `line:` narrows the plan to one line's rows (the per-line release + # workflows each plan their own). + def builds(line: nil) + rows_for(:build, line).map do |row| entry = @versions.fetch(row[:version]) row.merge(tree: entry.src_tree_name, asset: asset_name(row)) end end # Copy rows: one per unchanged (version x scenario build) asset. - def copies - rows_for(:copy).map { |row| row.slice(:version, :suffix).merge(asset: asset_name(row)) } + def copies(line: nil) + rows_for(:copy, line).map { |row| row.slice(:version, :suffix).merge(asset: asset_name(row)) } end private - def rows_for(action) - wanted = @versions.select { |entry| (action == :build) == build?(entry) }.map(&:name) - @versions.builds.select { |row| wanted.include?(row[:version]) } + def rows_for(action, line = nil) + @versions.builds.select do |row| + (line.nil? || @versions.fetch(row[:version]).line == line) && (action == :build) == build_row?(row) + end end def asset_name(row) "#{@versions.fetch(row[:version]).src_tree_name}#{row[:suffix]}.tar.gz" end - def build?(entry) + # The row-level decision: the version-level reasons (first release, + # shared tooling, the versions.yml entry moving) build every row of + # the version; otherwise the row builds iff the line's changed + # patches feed the row's scenario (and pass, when the change is + # pass-scoped). + def build_row?(row) return true if @diff.previous_tag.nil? return true if @diff.shared_change? - return true if @diff.patch_lines.include?(entry.line) + entry = @versions.fetch(row[:version]) previous = previous_entry(entry.name) - previous.nil? || state(previous) != state(entry) + return true if previous.nil? || state(previous) != state(entry) + + attributed_scenarios(entry.line).any? do |scenario, pass| + scenario == row[:platform] && (pass.nil? || pass == row[:pass]) + end + end + + # The line's changed-scenario list, failing CLOSED: a line the diff + # saw patch changes for but attributes nothing to (a shape the + # suffix rules do not produce today) feeds every scenario — never + # ship a possibly-stale copy. An explicitly EMPTY attribution (a + # darwin-only change — no shipped scenario) builds nothing. + def attributed_scenarios(line) + scenarios = @diff.changed_scenarios + return all_scenarios if scenarios.nil? + + rows = scenarios.fetch(line, nil) + return rows unless rows.nil? + + @diff.patch_lines&.include?(line) ? all_scenarios : [] + end + + def all_scenarios + Tfs::Versions::SCENARIOS.map { |scenario| [scenario, nil] } end def previous_entry(name) diff --git a/tools/lib/tfs/release_diff.rb b/tools/lib/tfs/release_diff.rb index 05af6b6..22192e2 100644 --- a/tools/lib/tfs/release_diff.rb +++ b/tools/lib/tfs/release_diff.rb @@ -91,6 +91,43 @@ def patch_lines changed_paths.filter_map { |path| path[LINE_PATH, 1] }.uniq end + # The scenario axis of the patch changes: which release scenarios each + # changed line's patches actually feed. The attribution follows the + # PatchSelection conventions (the single model of what lands where): + # + # * a terminal `_msys` / `_msys_` patch feeds the msys scenario + # only, and a `_pass1`/`_pass2` marker scopes it to that pass's + # tarball alone; + # * a terminal `_musl` patch feeds linux-musl only; + # * a terminal `_darwin` patch feeds NO shipped scenario today (no + # darwin scenario exists — the base tarball is the linux-gnu + # selection): it attributes to the empty set, and a darwin-only + # change re-rolls nothing. THAT IS A MODEL FACT, not an oversight: + # if a darwin scenario ever ships, extend SCENARIO_BUILDS and this + # map together; + # * a base patch (no platform suffix) feeds every scenario — even + # when an msys/musl sibling shadows it on one target, the base + # patch still lands on the others (fail closed, never narrow a + # base patch by shadowing); + # * a line manifest (patch-.yaml / patch-..yaml) + # re-scopes any feature in the line: all scenarios, fail closed. + # + # Returns { line => [[scenario, pass_or_nil], ...] }; nil means + # "everything" (no previous tag). A line absent from the map had no + # attributable patch change. + def changed_scenarios + return nil if previous_tag.nil? + + changed_paths.each_with_object({}) do |path, acc| + m = path.match(%r{\Apatches/(\d+\.\d+)/(.+)}) + next unless m + + line, name = m.captures + acc[line] ||= [] + acc[line] |= scenario_attribution(name) + end + end + # A shared input changed (or there is no previous tag): every line # rebuilds. Only patches//** and versions.yml are attributable # inputs; anything else — tools/**, ci/**, schema/**, workflows, a @@ -126,5 +163,33 @@ def release_tags def shared_path?(path) !(path.match?(LINE_PATH) || path == VERSIONS_MANIFEST) end + + ALL_PASSES = [nil].freeze + SCENARIO_SUFFIX = /_(msys|musl|darwin)(?:_\d+)?\.patch\z/ + PASS_MARKER = /_pass([12])(?:_|\z)/ + + # One patch path's [scenario, pass] attribution (the class doc's + # rules). `pass` is nil unless a _pass1/_pass2 marker scopes the + # patch; the msys-only pass markers never narrow a POSIX scenario. + def scenario_attribution(name) + if name.end_with?(".yaml") + return all_scenarios + end + + pass = name[PASS_MARKER, 1]&.to_i + case name.match(SCENARIO_SUFFIX)&.[](1) + when "msys" then [["msys", pass]] + when "musl" then [["linux-musl", nil]] + when "darwin" then [] + else + # A base patch: every scenario; a pass marker on a base patch is + # a manifest authoring error the lint gate owns — attribute wide. + all_scenarios + end + end + + def all_scenarios + Tfs::Versions::SCENARIOS.map { |scenario| [scenario, nil] } + end end end diff --git a/tools/lib/tfs/smoke_plan.rb b/tools/lib/tfs/smoke_plan.rb new file mode 100644 index 0000000..01f8137 --- /dev/null +++ b/tools/lib/tfs/smoke_plan.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +module Tfs + # The compile-smoke leg plan (release-src's publish gate): one + # representative leg per (changed line x affected scenario) at the + # line's NEWEST version, computing from the same Tfs::ReleaseDiff the + # build/copy plan uses. + # + # Two fault-isolation axes, both from the diff: + # + # * line: only lines whose patch set changed smoke at all; + # * scenario: only the scenarios the changed patches FEED smoke + # (Tfs::ReleaseDiff#changed_scenarios — an msys patch never smokes + # linux). A line whose attribution is explicitly empty (a darwin-only + # change — no shipped scenario) smokes nothing. + # + # A line-wide patch set means a representative leg per (line, scenario) + # covers the changed translation units; per-version legs would multiply + # configure runs for no extra signal. msys legs compile at pass 2 (the + # pass split selects GNUmakefile variants — no patched .c differs + # between passes). + class SmokePlan + # versions: Tfs::Versions of the tag being published. + # diff: Tfs::ReleaseDiff for the tag being published. + # line: optional line filter ("3.4") — one line's legs only. + def initialize(versions:, diff:, line: nil) + @versions = versions + @diff = diff + @line = line + end + + # The leg rows: {line:, version:, platform:} per (changed line x + # affected scenario), the newest version of each. + def legs + per_line = @versions.flat_map do |entry| + next [] unless include_entry?(entry) + + affected_scenarios(entry).map { |scenario| { line: entry.line, version: entry.name, platform: scenario } } + end + per_line.group_by { |leg| [leg[:line], leg[:platform]] } + .map { |(_line, _platform), group| group.max_by { |leg| Gem::Version.new(leg[:version]) } } + end + + private + + def include_entry?(entry) + return false if @line && entry.line != @line + return true if @diff.patch_lines.nil? + + @diff.patch_lines.include?(entry.line) + end + + def affected_scenarios(entry) + scenarios = @diff.changed_scenarios + return entry.scenarios if scenarios.nil? + + attributed = scenarios.fetch(entry.line, []) + entry.scenarios & attributed.map(&:first).uniq + end + end +end diff --git a/tools/smoke_matrix b/tools/smoke_matrix index 3df6062..3a6cc77 100755 --- a/tools/smoke_matrix +++ b/tools/smoke_matrix @@ -1,25 +1,21 @@ #!/usr/bin/env ruby # frozen_string_literal: true -# Usage: tools/smoke_matrix +# Usage: tools/smoke_matrix [--line X.Y] +# # Prints the release-src compile-smoke matrix as a GitHub Actions matrix -# document: one leg per (changed line x scenario), where a "changed line" -# is a patches// folder touched between the previous release tag and -# (or HEAD when the tag does not exist yet, e.g. a manual -# dispatch ahead of tagging; all lines when there is no previous tag). -# The diff itself is Tfs::ReleaseDiff — the same previous-tag base the -# build/copy plan (tools/build_matrix) uses; the smoke gate keys on patch -# sets only, since shared tooling does not change what a patched .c -# compiles to. +# document: one leg per (changed line x AFFECTED scenario) — only the +# scenarios the changed patches feed (an msys patch never smokes linux), +# where a "changed line" is a patches// folder touched between the +# previous release tag and (or HEAD when the tag does not +# exist yet, e.g. a manual dispatch ahead of tagging; all lines when +# there is no previous tag). The diff itself is Tfs::ReleaseDiff — the +# same previous-tag base the build/copy plan (tools/build_matrix) uses; +# the smoke gate keys on patch sets only, since shared tooling does not +# change what a patched .c compiles to. The leg plan is Tfs::SmokePlan. # -# Per-version legs (version x scenario of every changed patch set) would -# multiply configure runs for no extra signal: a patch is line-wide, so a -# REPRESENTATIVE leg per line -- the newest version of the changed line, -# per scenario it ships -- covers the changed translation units (documented -# in .github/workflows/release-src.yml). Each leg compiles every patched -# translation unit of that version for the scenario (tools/compile_smoke); -# msys legs run at pass 2 only (the pass split selects GNUmakefile -# variants -- no patched .c differs between passes). +# --line narrows the plan to one line (the per-line release workflows +# each plan their own). # # Emits {"include":[]} when no patch set changed: the gate is vacuously # green (an empty include matrix runs no legs and succeeds). @@ -28,18 +24,8 @@ $LOAD_PATH.unshift(File.expand_path("lib", __dir__)) require "tfs" require "json" -tag = ARGV[0] or abort "usage: tools/smoke_matrix " - -lines = Tfs::ReleaseDiff.new(tag).patch_lines - -versions = Tfs::Versions.new -legs = - versions.flat_map do |entry| - next [] unless lines.nil? || lines.include?(entry.line) - - entry.scenarios.map { |scenario| { line: entry.line, version: entry.name, platform: scenario } } - end -newest = legs.group_by { |leg| [leg[:line], leg[:platform]] } - .map { |(_line, _platform), group| group.max_by { |leg| Gem::Version.new(leg[:version]) } } +tag = ARGV[0] or abort "usage: tools/smoke_matrix [--line X.Y]" +line = ARGV[1] == "--line" ? ARGV[2] : nil -puts JSON.generate({ include: newest }) +legs = Tfs::SmokePlan.new(versions: Tfs::Versions.new, diff: Tfs::ReleaseDiff.new(tag), line: line).legs +puts JSON.generate({ include: legs })