From ab85f15aa4ed9668394c01483b785afdf579f547 Mon Sep 17 00:00:00 2001 From: David Ahmann <46606159+davidahmann@users.noreply.github.com> Date: Sat, 19 Sep 2026 16:50:41 -0400 Subject: [PATCH] fix: reconcile published releases without repeating npm publication Signed-off-by: David Ahmann <46606159+davidahmann@users.noreply.github.com> --- .github/workflows/release-recovery.yml | 286 +++++++++ AGENTS.md | 6 + docs/release.md | 44 +- product/admission-evidence-hardening.md | 19 + scripts/release-recovery.mjs | 585 +++++++++++++++++ scripts/retry-npm-metadata.mjs | 4 +- test/release-channel-readback.test.ts | 10 +- test/release-recovery.test.ts | 812 ++++++++++++++++++++++++ 8 files changed, 1755 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/release-recovery.yml create mode 100644 scripts/release-recovery.mjs create mode 100644 test/release-recovery.test.ts diff --git a/.github/workflows/release-recovery.yml b/.github/workflows/release-recovery.yml new file mode 100644 index 0000000..6ee105c --- /dev/null +++ b/.github/workflows/release-recovery.yml @@ -0,0 +1,286 @@ +name: Release recovery +run-name: Recover ${{ inputs.tag }} after npm publication + +on: + workflow_dispatch: + inputs: + tag: + description: Existing annotated stable tag already published to npm + required: true + type: string + candidate_run_id: + description: Successful candidate run consumed by original publication + required: true + type: string + publish_run_id: + description: Failed release run with one successful immutable npm step + required: true + type: string + +permissions: + actions: read + contents: read + +concurrency: + group: release-${{ inputs.tag }} + cancel-in-progress: false + +jobs: + finalize: + name: finalize-existing-npm-release + if: + github.ref == 'refs/heads/main' && github.repository == 'davidahmann/mill' + runs-on: ubuntu-24.04 + timeout-minutes: 60 + environment: npm + permissions: + actions: read + contents: write + defaults: + run: + working-directory: release-source + shell: bash + env: + GH_REPO: ${{ github.repository }} + MILL_NPM_METADATA_ATTEMPTS: "20" + MILL_NPM_METADATA_DELAY_MS: "30000" + steps: + - name: Checkout approved recovery controller + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ github.sha }} + path: controller + fetch-depth: 1 + persist-credentials: false + - name: Verify main controller identity + working-directory: controller + run: | + test "$GITHUB_REF" = refs/heads/main + test "$GITHUB_RUN_ATTEMPT" = 1 + test "$GITHUB_WORKFLOW_SHA" = "$GITHUB_SHA" + test "$(git rev-parse HEAD)" = "$GITHUB_SHA" + - name: Checkout immutable release source separately + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: ${{ inputs.tag }} + path: release-source + fetch-depth: 0 + persist-credentials: false + - name: Verify annotated tag and release notes + env: + RELEASE_TAG: ${{ inputs.tag }} + run: | + printf '%s\n' "$RELEASE_TAG" | grep -Eq '^v[0-9]+\.[0-9]+\.[0-9]+$' + normalized_ref=$(git check-ref-format --normalize "refs/tags/$RELEASE_TAG") + test "$normalized_ref" = "refs/tags/$RELEASE_TAG" + test "$(git cat-file -t "$normalized_ref")" = tag + test "$(git rev-parse "${normalized_ref}^{commit}")" = "$(git rev-parse HEAD^{commit})" + notes_file="docs/releases/${RELEASE_TAG}.md" + test -d docs && test ! -L docs && test -d docs/releases && test ! -L docs/releases + test -f "$notes_file" && test ! -L "$notes_file" && test -s "$notes_file" + - name: Set up Node and npm registry + uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version-file: release-source/.node-version + registry-url: https://registry.npmjs.org + cache: npm + cache-dependency-path: release-source/package-lock.json + - name: Observe original publication and require absent GitHub Release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + CANDIDATE_RUN_ID: ${{ inputs.candidate_run_id }} + PUBLISH_RUN_ID: ${{ inputs.publish_run_id }} + run: | + node "$GITHUB_WORKSPACE/controller/scripts/release-recovery.mjs" observe \ + "$RELEASE_TAG" "$CANDIDATE_RUN_ID" "$PUBLISH_RUN_ID" "$RUNNER_TEMP" + - name: Install tagged tools without lifecycle scripts + run: npm ci --ignore-scripts + - name: Build tagged release tools + run: npm run build + - name: Verify tagged identity without replacing recovery identity + env: + MILL_RELEASE_TAG: ${{ inputs.tag }} + run: | + tag_commit=$(git rev-parse HEAD^{commit}) + published_commit=$(node -e 'process.stdout.write(JSON.parse(require("node:fs").readFileSync(process.env.RUNNER_TEMP+"/publish-run.json","utf8")).head_sha)') + test "$tag_commit" = "$published_commit" + GITHUB_SHA="$tag_commit" node scripts/verify-release-tag.mjs > "$RUNNER_TEMP/identity.json" + - name: Download the qualified candidate run + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: genesis-candidate-${{ inputs.tag }} + path: ${{ runner.temp }}/qualified + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ inputs.candidate_run_id }} + - name: Download independent verifier receipt + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: independent-policy-${{ inputs.tag }} + path: ${{ runner.temp }}/trusted + github-token: ${{ github.token }} + repository: ${{ github.repository }} + run-id: ${{ inputs.candidate_run_id }} + - name: + Bind original candidate, publication and separate recovery controller + run: | + node scripts/verify-independent-release.mjs \ + "$RUNNER_TEMP/qualified" "$RUNNER_TEMP/trusted/trusted-verifier.json" \ + "$RUNNER_TEMP/candidate-run.json" "$RUNNER_TEMP/candidate-jobs.json" "$RUNNER_TEMP/identity.json" + node "$GITHUB_WORKSPACE/controller/scripts/release-recovery.mjs" validate \ + "$RUNNER_TEMP" "$RUNNER_TEMP/identity.json" "$RUNNER_TEMP/qualified/artifact-metadata.json" + - name: Verify preserved public-alpha qualification + run: | + qualification_input=.mill-release-qualification.json + test ! -e "$qualification_input" + node -e 'require("node:fs").copyFileSync(process.argv[1],process.argv[2],require("node:fs").constants.COPYFILE_EXCL)' \ + "$RUNNER_TEMP/qualified/qualification.json" "$qualification_input" + trap 'rm -f "$qualification_input"' EXIT + node dist/cli.js --json --cwd . qualify public-alpha \ + --file "$qualification_input" + rm -f "$qualification_input" + trap - EXIT + - name: Bind qualification and prepublication release evidence + run: | + node scripts/assemble-release-evidence.mjs \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/qualified/qualification.json" \ + "$RUNNER_TEMP/qualified/sbom.cdx.json" \ + "$RUNNER_TEMP/identity.json" \ + "$RUNNER_TEMP/qualified/release-evidence-prepublication.json" \ + - - "$RUNNER_TEMP/workflow-runs.json" + - name: Prepare the pinned verifier before GitHub finalization + id: prepare-release-verifier + run: | + verifier_image=$(node -e 'const fs=require("node:fs"),y=require("yaml");const image=y.parse(fs.readFileSync("recipes/node-typescript-next-web/recipe.yaml","utf8")).verifierImage;if(typeof image!=="string"||!/@sha256:[a-f0-9]{64}$/.test(image))throw new Error("Digest-pinned verifier required");process.stdout.write(image)') + docker pull "$verifier_image" + docker image inspect "$verifier_image" >/dev/null + - name: Read back npm and verify signatures + run: | + version=$(node -p 'require("./package.json").version') + node scripts/retry-npm-metadata.mjs \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/registry-dist.json" \ + "$RUNNER_TEMP/npm-observation-initial.json" + mkdir -p "$RUNNER_TEMP/registry-install" + cd "$RUNNER_TEMP/registry-install" + npm init --yes >/dev/null + node "$GITHUB_WORKSPACE/release-source/scripts/retry-npm-install.mjs" \ + "$RUNNER_TEMP/registry-install" "@davidahmann/mill@$version" + node "$GITHUB_WORKSPACE/release-source/scripts/retry-npm-signatures.mjs" "$RUNNER_TEMP/registry-install" + - name: Requalify the registry-downloaded package + run: | + version=$(node -p 'require("./package.json").version') + mkdir -p "$RUNNER_TEMP/registry-download" + npm pack "@davidahmann/mill@$version" --pack-destination "$RUNNER_TEMP/registry-download" >/dev/null + registry_artifact=$(find "$RUNNER_TEMP/registry-download" -maxdepth 1 -type f -name '*.tgz') + node "$GITHUB_WORKSPACE/controller/scripts/release-recovery.mjs" registry "$RUNNER_TEMP/qualified/artifact-metadata.json" "$RUNNER_TEMP/registry-download" + cd "$RUNNER_TEMP" + node "$GITHUB_WORKSPACE/release-source/scripts/qualify-release-artifact.mjs" "$registry_artifact" --full-canary + - name: Create draft GitHub Release with exact artifacts + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + artifact=$(find "$RUNNER_TEMP/qualified" -maxdepth 1 -type f -name '*.tgz') + checksum=$(find "$RUNNER_TEMP/qualified" -maxdepth 1 -type f -name '*.sha256') + notes_file="docs/releases/${RELEASE_TAG}.md" + test -f "$notes_file" + gh api --paginate --slurp "repos/$GITHUB_REPOSITORY/releases?per_page=100" > "$RUNNER_TEMP/releases-before-create.json" + node "$GITHUB_WORKSPACE/controller/scripts/release-recovery.mjs" absent "$RUNNER_TEMP/releases-before-create.json" "$RELEASE_TAG" + gh release create "$RELEASE_TAG" --verify-tag --draft --title "Mill $RELEASE_TAG (Public alpha)" \ + --notes-file "$notes_file" \ + "$artifact" "$checksum" \ + "$RUNNER_TEMP/qualified/sbom.cdx.json" \ + "$RUNNER_TEMP/qualified/release-evidence-prepublication.json" \ + "$RUNNER_TEMP/qualified/qualification.json" \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/qualified/audit.json" \ + "$RUNNER_TEMP/qualified/identity.json" \ + "$RUNNER_TEMP/qualified/release-canary.json" \ + "$RUNNER_TEMP/trusted/trusted-verifier.json" \ + "$RUNNER_TEMP/trusted/trusted-canary.json" + - name: Read back draft GitHub Release evidence + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + artifact=$(find "$RUNNER_TEMP/qualified" -maxdepth 1 -type f -name '*.tgz') + filename=$(basename "$artifact") + mkdir -p "$RUNNER_TEMP/github-download" + gh release view "$RELEASE_TAG" --json url,tagName,isDraft,isPrerelease,publishedAt,databaseId,assets > "$RUNNER_TEMP/github-release.json" + node -e 'const fs=require("node:fs");const file=process.argv[1];const release=JSON.parse(fs.readFileSync(file,"utf8"));release.url=`${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/releases/tag/${encodeURIComponent(process.env.RELEASE_TAG)}`;fs.writeFileSync(file,`${JSON.stringify(release)}\n`,{flag:"w",mode:0o600})' "$RUNNER_TEMP/github-release.json" + gh release download "$RELEASE_TAG" --pattern "$filename" --dir "$RUNNER_TEMP/github-download" + node scripts/capture-release-readback.mjs \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/registry-dist.json" \ + "$RUNNER_TEMP/github-release.json" \ + "$RUNNER_TEMP/github-download/$filename" \ + "$RUNNER_TEMP/registry-readback.json" \ + "$RUNNER_TEMP/github-readback.json" + node scripts/assemble-release-evidence.mjs \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/qualified/qualification.json" \ + "$RUNNER_TEMP/qualified/sbom.cdx.json" \ + "$RUNNER_TEMP/identity.json" \ + "$RUNNER_TEMP/release-evidence-draft.json" \ + "$RUNNER_TEMP/registry-readback.json" \ + "$RUNNER_TEMP/github-readback.json" \ + "$RUNNER_TEMP/workflow-runs.json" + gh release upload "$RELEASE_TAG" "$RUNNER_TEMP/release-evidence-draft.json" + - name: Publish GitHub Release after draft evidence readback + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + gh release edit "$RELEASE_TAG" --draft=false --latest + - name: Read back published GitHub Release and attach final evidence + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ inputs.tag }} + run: | + artifact=$(find "$RUNNER_TEMP/qualified" -maxdepth 1 -type f -name '*.tgz') + filename=$(basename "$artifact") + rm -rf "$RUNNER_TEMP/github-download" + mkdir -p "$RUNNER_TEMP/github-download" + node scripts/retry-npm-metadata.mjs \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/registry-dist-final.json" \ + "$RUNNER_TEMP/npm-observation-final.json" + gh release view "$RELEASE_TAG" --json url,tagName,isDraft,isPrerelease,publishedAt,databaseId,assets > "$RUNNER_TEMP/github-release-published.json" + node -e 'const fs=require("node:fs");const file=process.argv[1];const release=JSON.parse(fs.readFileSync(file,"utf8"));release.url=`${process.env.GITHUB_SERVER_URL}/${process.env.GITHUB_REPOSITORY}/releases/tag/${encodeURIComponent(process.env.RELEASE_TAG)}`;fs.writeFileSync(file,`${JSON.stringify(release)}\n`,{flag:"w",mode:0o600})' "$RUNNER_TEMP/github-release-published.json" + gh release download "$RELEASE_TAG" --pattern "$filename" --dir "$RUNNER_TEMP/github-download" + node scripts/capture-release-readback.mjs \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/registry-dist-final.json" \ + "$RUNNER_TEMP/github-release-published.json" \ + "$RUNNER_TEMP/github-download/$filename" \ + "$RUNNER_TEMP/registry-readback-published.json" \ + "$RUNNER_TEMP/github-readback-published.json" + gh api "repos/$GITHUB_REPOSITORY/releases/latest" > "$RUNNER_TEMP/github-latest.json" + node scripts/capture-release-channels.mjs \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/npm-observation-final.json" \ + "$RUNNER_TEMP/github-latest.json" \ + "$RUNNER_TEMP/github-readback-published.json" \ + "$RUNNER_TEMP/release-channels.json" + node scripts/assemble-release-evidence.mjs \ + "$RUNNER_TEMP/qualified/artifact-metadata.json" \ + "$RUNNER_TEMP/qualified/qualification.json" \ + "$RUNNER_TEMP/qualified/sbom.cdx.json" \ + "$RUNNER_TEMP/identity.json" \ + "$RUNNER_TEMP/release-evidence-final.json" \ + "$RUNNER_TEMP/registry-readback-published.json" \ + "$RUNNER_TEMP/github-readback-published.json" \ + "$RUNNER_TEMP/workflow-runs.json" \ + "$RUNNER_TEMP/release-channels.json" + gh release upload "$RELEASE_TAG" "$RUNNER_TEMP/release-evidence-final.json" + node "$GITHUB_WORKSPACE/controller/scripts/release-recovery.mjs" finalize \ + "$RUNNER_TEMP" "$RUNNER_TEMP/release-evidence-final.json" "$RUNNER_TEMP/qualified/qualification.json" + gh release upload "$RELEASE_TAG" "$RUNNER_TEMP/release-recovery.json" "$RUNNER_TEMP/recovery-observations.json" + mkdir "$RUNNER_TEMP/recovery-readback" + gh release download "$RELEASE_TAG" --pattern release-recovery.json --pattern release-evidence-final.json --pattern recovery-observations.json --dir "$RUNNER_TEMP/recovery-readback" + cmp "$RUNNER_TEMP/release-recovery.json" "$RUNNER_TEMP/recovery-readback/release-recovery.json" + cmp "$RUNNER_TEMP/release-evidence-final.json" "$RUNNER_TEMP/recovery-readback/release-evidence-final.json" + cmp "$RUNNER_TEMP/recovery-observations.json" "$RUNNER_TEMP/recovery-readback/recovery-observations.json" diff --git a/AGENTS.md b/AGENTS.md index e490f81..a6efcd3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -320,6 +320,12 @@ Reconcile exact assets and obtain owner disposition before changing an existing release. Preserve failed runs, tags and artifact bytes. A later npm dist-tag or GitHub Latest change needs authoritative readback, not a rebuild. +The protected finalization-only recovery workflow runs from reviewed `main` and +separately verifies the original tag, qualified candidate and successful npm +publish step. It cannot publish npm or overwrite an existing GitHub Release. +Retain the original failed workflow as evidence and bind the recovery source +separately. Follow the narrower recovery procedure in `docs/release.md`. + ## Engineering rules - Node.js 24.20.0, TypeScript, ESM, strict types, and exact dependency pins. diff --git a/docs/release.md b/docs/release.md index e54b2f5..18fdd17 100644 --- a/docs/release.md +++ b/docs/release.md @@ -217,11 +217,15 @@ It does not rebuild or repack the local candidate. Bounded read-only retries wait for npm's exact version, integrity, provenance metadata and `latest` tag, then package installation and signature verification. A conflicting immutable version or integrity fails immediately; publication itself is never retried. The -job downloads and requalifies the registry artifact, creates a plainly labelled -draft public-alpha release with the same tarball/checksum/SBOM/evidence, and -verifies the downloaded GitHub asset. It uploads draft evidence before -publishing the normal GitHub Release with `--latest`. Fresh npm and GitHub -Latest readbacks then supply the final evidence, which is uploaded last. +metadata reader allows twenty attempts thirty seconds apart, with a separate +sixty-second limit per request. The v0.7.1 publication exposed a propagation +delay beyond the previous twelve-attempt, ten-second interval. The wider budget +changes only reads; it grants no retry of publication. The job downloads and +requalifies the registry artifact, creates a plainly labelled draft public-alpha +release with the same tarball/checksum/SBOM/evidence, and verifies the +downloaded GitHub asset. It uploads draft evidence before publishing the normal +GitHub Release with `--latest`. Fresh npm and GitHub Latest readbacks then +supply the final evidence, which is uploaded last. ### 5. Close the release @@ -299,6 +303,29 @@ architecture follow-through canary evidence. ## Withdrawal +### Recover publication before GitHub Release creation + +If npm publication succeeded but metadata readback exhausted its budget, do not +rerun the publish job. The protected `release-recovery.yml` workflow can finish +the narrower case where the original publish step succeeded, the candidate and +trusted-verifier artifacts remain available, and no GitHub Release exists for +the tag. Dispatch the reviewed recovery workflow from `main` with the exact tag, +candidate run ID and original publish run ID. + +Recovery verifies those identities and the existing npm version, signatures and +provenance, then runs the registry artifact's full canary before creating a +draft. Final evidence retains the original publish run. A separate recovery +record identifies the recovery workflow and its source commit; a green recovery +does not rewrite the original failed run. Any existing release, ambiguous +creation result, missing artifact or identity conflict blocks this route. + +The v0.7.1 incident used candidate `35466952501` and original publish +`35467443471`. Its successful immutable publish step preceded metadata +propagation; no release had been created. Its tag and npm bytes remain +unchanged. + +### Other interrupted or unsafe releases + First classify an interrupted publication from provider readback; do not rerun the publish workflow. If npm and registry qualification succeeded but GitHub finalization is incomplete, inventory all release records for the tag by numeric @@ -316,7 +343,12 @@ If prepublication qualification fails, do not publish. Delete an unpushed local tag, fix through a new reviewed PR, and restart with a new exact candidate. If a remote tag exists, retain it as evidence and use a new version. -If publication succeeded but readback or postpublication qualification fails: +If provider readback is unavailable, stop further effects and reconcile it. A +verified existing publication with no GitHub Release may use the recovery route +above. A timeout alone is not proof that the package is unsafe. + +If readback proves an artifact or identity mismatch, or the published artifact +fails qualification: 1. stop the workflow before making a public support claim; 2. deprecate the exact npm version with a concise safety message; diff --git a/product/admission-evidence-hardening.md b/product/admission-evidence-hardening.md index 945a07e..3d5eb44 100644 --- a/product/admission-evidence-hardening.md +++ b/product/admission-evidence-hardening.md @@ -52,3 +52,22 @@ No new support tuple, customer operation, downstream product implementation, daemon, parallel runtime writer, deployment, credential sharing or review bypass is authorized. Product readiness remains conditional on external evidence and owner decisions; software tests cannot close those gates. + +## Observed publication recovery + +On 2026-09-19, candidate run `35466952501` passed all four release gates for tag +`v0.7.1`, commit `50873b47574e9a46ca009c514186532332f385eb`. Publish run +`35467443471` completed the immutable npm publication, then exhausted its twelve +metadata reads before npm exposed the version. Subsequent registry readback +confirmed the exact artifact integrity, provenance and `latest` pointer. No +GitHub Release was created by that run. + +The owner's end-to-end repair and release instruction includes completing this +known publication and fixing the observed recovery gap. Extend the bounded read +window and add a protected, finalization-only workflow. It must prove the +original successful publish step and qualified artifact, verify the existing +registry version, and preserve the original publish run in release evidence. +Record the recovery workflow's separate source identity. The first recovery +route requires the GitHub tag's release to be absent; an existing or ambiguous +release blocks it. Never repeat publication, replace artifacts, move the tag, +invent a successful original run, or bypass the protected environment reviewer. diff --git a/scripts/release-recovery.mjs b/scripts/release-recovery.mjs new file mode 100644 index 0000000..c2b7169 --- /dev/null +++ b/scripts/release-recovery.mjs @@ -0,0 +1,585 @@ +import { execFileSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile, readdir, writeFile } from "node:fs/promises"; +import path from "node:path"; + +const repository = "davidahmann/mill"; +const workflowId = 346766046; +const workflowPath = ".github/workflows/release.yml"; +const recoveryPath = ".github/workflows/release-recovery.yml"; +const publishName = "Publish the exact preserved artifact with provenance"; +const preparationNames = [ + "Checkout immutable tag", + "Verify immutable release dispatch", + "Validate publish inputs", + "Set up Node and npm registry", + "Install without lifecycle scripts", + "Build release tools", + "Verify release identity", + "Download the qualified candidate run", + "Download independent verifier receipt", + "Bind candidate run and independently verified artifact", + "Bind candidate and publish workflow identities", + "Verify preserved public-alpha qualification", + "Bind qualification and prepublication release evidence", + "Prepare the pinned verifier before immutable publication", + publishName, +]; +const githubNames = [ + "Create draft GitHub Release with exact artifacts", + "Read back draft GitHub Release evidence", + "Publish GitHub Release after draft evidence readback", + "Read back published GitHub Release and attach final evidence", +]; +const demand = (condition, message) => { + if (!condition) throw new Error(message); +}; +const numericId = (value) => /^[1-9][0-9]*$/u.test(String(value)); +const sha = (value) => + typeof value === "string" && /^[a-f0-9]{40}$/u.test(value); +const timestamp = (value) => + typeof value === "string" && Number.isFinite(Date.parse(value)); +const read = async (file) => JSON.parse(await readFile(file, "utf8")); +const write = (file, value) => + writeFile(file, `${JSON.stringify(value, null, 2)}\n`, { + flag: "wx", + mode: 0o600, + }); +const digest = (bytes) => + `sha256:${createHash("sha256").update(bytes).digest("hex")}`; + +function pageItems(pages, key) { + demand(Array.isArray(pages) && pages.length > 0, `missing ${key} pages`); + const total = pages[0].total_count; + demand( + Number.isSafeInteger(total) && total >= 0 && total <= 1000, + `unbounded ${key} inventory`, + ); + demand( + pages.every( + (page) => page.total_count === total && Array.isArray(page[key]), + ), + `invalid ${key} pagination`, + ); + const items = pages.flatMap((page) => page[key]); + demand( + items.length === total && + new Set(items.map((item) => item.id)).size === total, + `incomplete or duplicate ${key} inventory`, + ); + return items; +} + +function assertAbsent(pages, tag) { + demand( + Array.isArray(pages) && pages.length > 0 && pages.every(Array.isArray), + "invalid release inventory", + ); + for (const release of pages.flat()) { + demand( + Number.isSafeInteger(release.id) && + release.id > 0 && + typeof release.tag_name === "string" && + typeof release.draft === "boolean", + "invalid release observation", + ); + demand( + release.tag_name !== tag, + "GitHub Release already exists for this tag; recovery cannot create or overwrite it", + ); + } +} + +function context(env) { + demand( + env.GITHUB_REPOSITORY === repository && + env.GITHUB_SERVER_URL === "https://github.com" && + env.GITHUB_EVENT_NAME === "workflow_dispatch" && + env.GITHUB_REF === "refs/heads/main" && + env.GITHUB_RUN_ATTEMPT === "1" && + numericId(env.GITHUB_RUN_ID) && + sha(env.GITHUB_SHA) && + env.GITHUB_WORKFLOW_SHA === env.GITHUB_SHA && + env.GITHUB_WORKFLOW_REF === + `${repository}/${recoveryPath}@refs/heads/main`, + "recovery requires a first-attempt dispatch of the exact main controller", + ); + return { + id: env.GITHUB_RUN_ID, + attempt: 1, + url: `https://github.com/${repository}/actions/runs/${env.GITHUB_RUN_ID}`, + headCommit: env.GITHUB_SHA, + workflowPath: recoveryPath, + ref: env.GITHUB_REF, + }; +} + +function history(input) { + const { + tag, + candidateId, + publishId, + recoveryRun, + workflow, + runsPages, + jobsByRun, + } = input; + demand( + /^v\d+\.\d+\.\d+$/u.test(tag) && + numericId(candidateId) && + numericId(publishId) && + candidateId !== publishId, + "invalid stable tag or distinct run inputs", + ); + demand( + workflow.id === workflowId && + workflow.path === workflowPath && + workflow.state === "active", + "unexpected release workflow identity", + ); + const runs = pageItems(runsPages, "workflow_runs"); + demand(runs.length <= 50, "too many release attempts for bounded recovery"); + const publish = runs.find((run) => String(run.id) === publishId); + const candidate = runs.find((run) => String(run.id) === candidateId); + demand( + publish !== undefined && + candidate !== undefined && + publishId !== recoveryRun.id && + candidateId !== recoveryRun.id, + "missing distinct candidate or original publish run", + ); + for (const run of runs) { + demand( + run.repository?.full_name === repository && + run.workflow_id === workflowId && + run.path === workflowPath && + run.event === "workflow_dispatch" && + run.head_branch === tag && + sha(run.head_sha) && + run.head_sha === publish.head_sha && + run.status === "completed" && + run.run_attempt === 1 && + run.html_url === + `https://github.com/${repository}/actions/runs/${run.id}`, + "release run has wrong identity, is unfinished, or has repeated attempts", + ); + const jobs = pageItems(jobsByRun[String(run.id)], "jobs"); + demand( + jobs.every( + (job) => + job.run_id === run.id && + job.run_attempt === 1 && + job.head_sha === run.head_sha && + job.head_branch === tag && + job.status === "completed", + ), + "job does not belong to exact completed first attempt", + ); + const publishers = jobs.filter( + (job) => job.name === "publish-qualified-artifact", + ); + demand(publishers.length === 1, "expected exactly one publish job per run"); + if (run.id !== publish.id) + demand( + publishers[0].conclusion === "skipped" && + publishers[0].steps.length === 0, + "another publish attempt exists; reconcile it before recovery", + ); + if (run.id === candidate.id) { + demand(run.conclusion === "success", "candidate did not succeed"); + for (const name of [ + "build-a", + "build-b", + "qualify-candidate", + "independent-release-policy", + ]) { + const matches = jobs.filter((job) => job.name === name); + demand( + matches.length === 1 && matches[0].conclusion === "success", + `missing successful candidate job: ${name}`, + ); + } + } + } + const jobs = pageItems(jobsByRun[publishId], "jobs"); + const job = jobs.find((item) => item.name === "publish-qualified-artifact"); + demand( + publish.conclusion === "failure" && + job.conclusion === "failure" && + jobs.every( + (item) => + item === job || + (item.conclusion === "skipped" && item.steps.length === 0), + ), + "recovery only accepts a failed publish-mode run", + ); + demand( + Array.isArray(job.steps) && + new Set(job.steps.map((step) => step.number)).size === job.steps.length, + "ambiguous publish steps", + ); + const step = (name, conclusion) => { + const matches = job.steps.filter((item) => item.name === name); + demand( + matches.length === 1 && + matches[0].status === "completed" && + matches[0].conclusion === conclusion && + Number.isSafeInteger(matches[0].number) && + matches[0].number > 0 && + timestamp(matches[0].started_at) && + timestamp(matches[0].completed_at) && + Date.parse(matches[0].started_at) <= + Date.parse(matches[0].completed_at), + `missing exact ${conclusion} step: ${name}`, + ); + return matches[0]; + }; + let prior; + for (const name of preparationNames) { + const current = step(name, "success"); + demand( + prior === undefined || + (current.number > prior.number && + Date.parse(current.started_at) >= Date.parse(prior.completed_at)), + "publish preparation order is invalid", + ); + prior = current; + } + const npmPublish = step(publishName, "success"); + const readback = step("Read back npm and verify signatures", "failure"); + demand( + readback.number > npmPublish.number && + Date.parse(readback.started_at) >= Date.parse(npmPublish.completed_at), + "npm readback did not fail after completed publication", + ); + for (const name of [ + "Requalify the registry-downloaded package", + ...githubNames, + ]) + demand( + step(name, "skipped").number > readback.number, + "GitHub effects were not skipped after readback failure", + ); + const known = new Set([ + ...preparationNames, + "Read back npm and verify signatures", + "Requalify the registry-downloaded package", + ...githubNames, + "Set up job", + "Post Set up Node and npm registry", + "Post Checkout immutable tag", + "Complete job", + ]); + demand( + job.steps.every((item) => known.has(item.name)), + "unknown original publish step requires manual reconciliation", + ); + assertAbsent(input.releasesPages, tag); + const runIdentity = (run) => ({ + id: String(run.id), + url: run.html_url, + headCommit: run.head_sha, + }); + return { + candidate, + publish, + job, + npmPublish, + workflowRuns: { + candidate: runIdentity(candidate), + publish: runIdentity(publish), + }, + }; +} + +function api(endpoint, paginated = false) { + const args = [ + "api", + ...(paginated ? ["--paginate", "--slurp"] : []), + `repos/${repository}/${endpoint}`, + ]; + return JSON.parse( + execFileSync("gh", args, { + encoding: "utf8", + timeout: 120_000, + maxBuffer: 8 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }), + ); +} + +// Only provider facts consumed by this gate become permanent public evidence. +const pick = (value, keys) => + Object.fromEntries(keys.map((key) => [key, value[key]])); +const runObservation = (value) => ({ + ...pick(value, [ + "id", + "workflow_id", + "path", + "event", + "head_branch", + "head_sha", + "status", + "run_attempt", + "html_url", + "conclusion", + ]), + repository: pick(value.repository, ["full_name"]), +}); +const jobObservation = (value) => ({ + ...pick(value, [ + "id", + "run_id", + "run_attempt", + "head_sha", + "head_branch", + "status", + "name", + "conclusion", + "html_url", + ]), + steps: value.steps.map((step) => + pick(step, [ + "name", + "number", + "status", + "conclusion", + "started_at", + "completed_at", + ]), + ), +}); +const observationPages = (pages, key, select) => + pages.map((page) => ({ + total_count: page.total_count, + [key]: page[key].map(select), + })); + +function candidateBinding(log, candidateId, tag, job) { + const lines = log + .split(/\r?\n/u) + .map((line) => line.replace(/^\d{4}-\d{2}-\d{2}T\S+ /u, "")); + const groups = []; + let group; + for (const line of lines) { + if (line.startsWith("##[group]")) { + demand(group === undefined, "nested job log groups are ambiguous"); + group = [line]; + } else if (line === "##[endgroup]") { + if (group !== undefined) groups.push(group); + group = undefined; + } else if (group !== undefined) group.push(line); + } + const command = + '##[group]Run gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CANDIDATE_RUN_ID" > "$RUNNER_TEMP/candidate-run.json"'; + const binding = groups.filter((item) => item[0] === command); + demand( + binding.length === 1 && + binding[0].filter((line) => line.startsWith(" CANDIDATE_RUN_ID:")) + .length === 1 && + binding[0].includes(` CANDIDATE_RUN_ID: ${candidateId}`), + "original publish consumed a different or unproven candidate run", + ); + for (const name of [ + `genesis-candidate-${tag}`, + `independent-policy-${tag}`, + ]) { + const downloads = groups.filter( + (item) => + item[0] === + "##[group]Run actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c" && + item.includes(` name: ${name}`), + ); + demand( + downloads.length === 1 && + downloads[0].filter((line) => line.startsWith(" run-id:")).length === + 1 && + downloads[0].includes(` run-id: ${candidateId}`) && + downloads[0].includes(` repository: ${repository}`), + "original artifact download did not consume the approved candidate run", + ); + } + return { + candidateId, + jobId: String(job.id), + logsUrl: `https://api.github.com/repos/${repository}/actions/jobs/${job.id}/logs`, + logsDigest: digest(log), + bindingStepName: "Bind candidate run and independently verified artifact", + }; +} + +const [mode, ...args] = process.argv.slice(2); +if (mode === "observe") { + const [tag, candidateId, publishId, directory] = args; + const recoveryRun = context(process.env); + demand( + /^v\d+\.\d+\.\d+$/u.test(tag) && + numericId(candidateId) && + numericId(publishId) && + typeof directory === "string", + "invalid recovery inputs", + ); + const workflow = pick(api("actions/workflows/release.yml"), [ + "id", + "path", + "state", + ]); + const runsPages = observationPages( + api( + `actions/workflows/release.yml/runs?event=workflow_dispatch&branch=${tag}&per_page=100`, + true, + ), + "workflow_runs", + runObservation, + ); + const runs = pageItems(runsPages, "workflow_runs"); + demand(runs.length <= 50, "too many release attempts for bounded recovery"); + const jobsByRun = {}; + for (const run of runs) { + demand( + numericId(run.id) && run.run_attempt === 1, + "repeated or invalid release attempt", + ); + jobsByRun[String(run.id)] = observationPages( + api(`actions/runs/${run.id}/attempts/1/jobs?per_page=100`, true), + "jobs", + jobObservation, + ); + } + const input = { + tag, + candidateId, + publishId, + recoveryRun, + workflow, + runsPages, + jobsByRun, + releasesPages: api("releases?per_page=100", true).map((page) => + page.map((release) => pick(release, ["id", "tag_name", "draft"])), + ), + observedAt: new Date().toISOString(), + }; + const accepted = history(input); + const log = execFileSync( + "gh", + ["api", `repos/${repository}/actions/jobs/${accepted.job.id}/logs`], + { + encoding: "utf8", + timeout: 120_000, + maxBuffer: 8 * 1024 * 1024, + stdio: ["ignore", "pipe", "pipe"], + }, + ); + candidateBinding(log, candidateId, tag, accepted.job); + await writeFile(path.join(directory, "publish-job.log"), log, { + flag: "wx", + mode: 0o600, + }); + await write(path.join(directory, "recovery-observations.json"), input); + await write(path.join(directory, "candidate-run.json"), accepted.candidate); + await write( + path.join(directory, "candidate-jobs.json"), + jobsByRun[candidateId], + ); + await write(path.join(directory, "publish-run.json"), accepted.publish); +} else if (mode === "validate") { + const [directory, identityFile, metadataFile] = args; + const input = await read(path.join(directory, "recovery-observations.json")); + demand( + JSON.stringify(input.recoveryRun) === JSON.stringify(context(process.env)), + "recovery controller context changed", + ); + const accepted = history(input); + const binding = candidateBinding( + await readFile(path.join(directory, "publish-job.log"), "utf8"), + input.candidateId, + input.tag, + accepted.job, + ); + const identity = await read(identityFile); + const metadata = await read(metadataFile); + demand( + identity.tag === input.tag && + identity.tagCommit === accepted.publish.head_sha && + sha(identity.mainTree) && + identity.mainTree === identity.reviewedCandidateTree && + identity.packageName === "@davidahmann/mill" && + identity.version === input.tag.slice(1) && + metadata.package.name === identity.packageName && + metadata.package.version === identity.version, + "recovery tag, package or original publication identity mismatch", + ); + await write( + path.join(directory, "workflow-runs.json"), + accepted.workflowRuns, + ); + await write(path.join(directory, "recovery-source.json"), { + schemaVersion: "1", + tag: input.tag, + source: identity, + artifactDigest: metadata.selectedArtifact.sha256, + workflowRuns: accepted.workflowRuns, + originalPublish: { + attempt: 1, + jobId: String(accepted.job.id), + jobUrl: accepted.job.html_url, + step: accepted.npmPublish, + candidateBinding: binding, + }, + recoveryRun: input.recoveryRun, + observationsDigest: digest( + await readFile(path.join(directory, "recovery-observations.json")), + ), + observedAt: input.observedAt, + }); +} else if (mode === "absent") { + const [pagesFile, tag] = args; + assertAbsent(await read(pagesFile), tag); +} else if (mode === "registry") { + const [metadataFile, directory] = args; + const metadata = await read(metadataFile); + const files = (await readdir(directory)).filter((file) => + file.endsWith(".tgz"), + ); + demand(files.length === 1, "expected one registry tarball"); + const bytes = await readFile(path.join(directory, files[0])); + demand( + digest(bytes) === metadata.selectedArtifact.sha256 && + `sha512-${createHash("sha512").update(bytes).digest("base64")}` === + metadata.selectedArtifact.npmIntegrity, + "registry bytes differ from preserved artifact", + ); +} else if (mode === "finalize") { + const [directory, finalEvidenceFile, qualificationFile] = args; + const source = await read(path.join(directory, "recovery-source.json")); + demand( + JSON.stringify(source.recoveryRun) === JSON.stringify(context(process.env)), + "recovery controller context changed", + ); + const finalBytes = await readFile(finalEvidenceFile); + const evidence = JSON.parse(finalBytes); + demand( + evidence.state === "verified" && + evidence.githubRelease?.state === "published" && + evidence.package.tag === source.tag && + evidence.source.tagCommit === source.source.tagCommit && + evidence.source.resultingMainCommit === source.source.tagCommit && + evidence.source.resultingMainTree === source.source.mainTree && + evidence.source.reviewedCandidateTree === source.source.mainTree && + evidence.selectedArtifact.sha256 === source.artifactDigest && + JSON.stringify(evidence.workflowRuns) === + JSON.stringify(source.workflowRuns), + "final evidence differs from recovered publication", + ); + await write(path.join(directory, "release-recovery.json"), { + ...source, + qualificationFileDigest: digest(await readFile(qualificationFile)), + finalEvidenceFileDigest: digest(finalBytes), + releaseId: evidence.githubRelease.releaseId, + completedAt: new Date().toISOString(), + }); +} else { + throw new Error( + "expected observe, validate, absent, registry or finalize recovery operation", + ); +} diff --git a/scripts/retry-npm-metadata.mjs b/scripts/retry-npm-metadata.mjs index c536e14..9776325 100644 --- a/scripts/retry-npm-metadata.mjs +++ b/scripts/retry-npm-metadata.mjs @@ -33,14 +33,14 @@ function bounded(value, fallback, minimum, maximum, name) { } const attempts = bounded( process.env.MILL_NPM_METADATA_ATTEMPTS, - 12, + 20, 1, 20, "MILL_NPM_METADATA_ATTEMPTS", ); const delayMs = bounded( process.env.MILL_NPM_METADATA_DELAY_MS, - 10_000, + 30_000, 0, 60_000, "MILL_NPM_METADATA_DELAY_MS", diff --git a/test/release-channel-readback.test.ts b/test/release-channel-readback.test.ts index 4421473..60d222b 100644 --- a/test/release-channel-readback.test.ts +++ b/test/release-channel-readback.test.ts @@ -30,6 +30,7 @@ describe("bounded npm metadata and final channel readback", () => { { mode: "http-transient", expectedAttempts: 3, accepted: true }, { mode: "tag-transient", expectedAttempts: 3, accepted: true }, { mode: "provenance-transient", expectedAttempts: 3, accepted: true }, + { mode: "late-propagation", expectedAttempts: 13, accepted: true }, { mode: "integrity-conflict", expectedAttempts: 1, accepted: false }, { mode: "version-conflict", expectedAttempts: 1, accepted: false }, { mode: "unavailable", expectedAttempts: 3, accepted: false }, @@ -50,7 +51,7 @@ const mode = process.env.TEST_MODE; const countFile = process.env.TEST_COUNT; const count = fs.existsSync(countFile) ? Number(fs.readFileSync(countFile, "utf8")) + 1 : 1; fs.writeFileSync(countFile, String(count)); -if (mode === "unavailable" || (mode === "http-transient" && count < 3)) { process.stderr.write("npm error E404/503 propagation delay"); process.exit(1); } +if (mode === "unavailable" || (mode === "http-transient" && count < 3) || (mode === "late-propagation" && count < 13)) { process.stderr.write("npm error E404/503 propagation delay"); process.exit(1); } const response = { version: "0.7.0", dist: { integrity: ${JSON.stringify(integrity)}, tarball: "https://registry.npmjs.org/@davidahmann/mill/-/mill-0.7.0.tgz", attestations: { url: "https://registry.npmjs.org/attestations", provenance: { predicateType: "https://slsa.dev/provenance/v1" } } }, "dist-tags": { latest: "0.7.0" } }; if (mode === "integrity-conflict") response.dist.integrity = "sha512-other"; if (mode === "version-conflict") response.version = "0.6.0"; @@ -60,6 +61,10 @@ process.stdout.write(JSON.stringify(response)); `, { mode: 0o755 }, ); + const environment = { ...process.env }; + if (mode === "late-propagation") + delete environment.MILL_NPM_METADATA_ATTEMPTS; + else environment.MILL_NPM_METADATA_ATTEMPTS = "3"; const result = spawnSync( process.execPath, [ @@ -72,11 +77,10 @@ process.stdout.write(JSON.stringify(response)); encoding: "utf8", timeout: 10_000, env: { - ...process.env, + ...environment, PATH: `${temporary.path}${path.delimiter}${process.env.PATH ?? ""}`, TEST_MODE: mode, TEST_COUNT: file("count"), - MILL_NPM_METADATA_ATTEMPTS: "3", MILL_NPM_METADATA_DELAY_MS: "0", }, }, diff --git a/test/release-recovery.test.ts b/test/release-recovery.test.ts new file mode 100644 index 0000000..4954f34 --- /dev/null +++ b/test/release-recovery.test.ts @@ -0,0 +1,812 @@ +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import { describe, expect, it } from "vitest"; +import { parse } from "yaml"; + +import { temporaryDirectory } from "./helpers.js"; + +const script = path.resolve("scripts/release-recovery.mjs"); +const repository = "davidahmann/mill"; +const tag = "v0.7.1"; +const sourceCommit = "a".repeat(40); +const controllerCommit = "b".repeat(40); +const env = { + ...process.env, + GITHUB_REPOSITORY: repository, + GITHUB_SERVER_URL: "https://github.com", + GITHUB_EVENT_NAME: "workflow_dispatch", + GITHUB_REF: "refs/heads/main", + GITHUB_RUN_ATTEMPT: "1", + GITHUB_RUN_ID: "44", + GITHUB_SHA: controllerCommit, + GITHUB_WORKFLOW_SHA: controllerCommit, + GITHUB_WORKFLOW_REF: `${repository}/.github/workflows/release-recovery.yml@refs/heads/main`, +}; +const digest = (value: Buffer | string) => + `sha256:${createHash("sha256").update(value).digest("hex")}`; +function required(value: T | undefined): T { + if (value === undefined) throw new Error("missing test fixture"); + return value; +} +function fixture() { + const run = (id: number, conclusion: string) => ({ + id, + conclusion, + status: "completed", + repository: { full_name: repository }, + workflow_id: 346766046, + path: ".github/workflows/release.yml", + event: "workflow_dispatch", + head_branch: tag, + head_sha: sourceCommit, + run_attempt: 1, + html_url: `https://github.com/${repository}/actions/runs/${id}`, + }); + const stepNames = [ + "Set up job", + "Checkout immutable tag", + "Verify immutable release dispatch", + "Validate publish inputs", + "Set up Node and npm registry", + "Install without lifecycle scripts", + "Build release tools", + "Verify release identity", + "Download the qualified candidate run", + "Download independent verifier receipt", + "Bind candidate run and independently verified artifact", + "Bind candidate and publish workflow identities", + "Verify preserved public-alpha qualification", + "Bind qualification and prepublication release evidence", + "Prepare the pinned verifier before immutable publication", + "Publish the exact preserved artifact with provenance", + "Read back npm and verify signatures", + "Requalify the registry-downloaded package", + "Create draft GitHub Release with exact artifacts", + "Read back draft GitHub Release evidence", + "Publish GitHub Release after draft evidence readback", + "Read back published GitHub Release and attach final evidence", + ]; + const steps = stepNames.map((name, index) => ({ + name, + number: index + 1, + status: "completed", + conclusion: index < 16 ? "success" : index === 16 ? "failure" : "skipped", + started_at: "2026-09-19T20:00:00Z", + completed_at: "2026-09-19T20:00:00Z", + })); + const job = ( + id: number, + runId: number, + name: string, + conclusion: string, + jobSteps = steps.slice(0, 0), + ) => ({ + id, + run_id: runId, + head_sha: sourceCommit, + head_branch: tag, + run_attempt: 1, + status: "completed", + conclusion, + name, + steps: jobSteps, + html_url: `https://github.com/${repository}/actions/runs/${runId}/job/${id}`, + }); + const candidateJobs = [ + job(420, 42, "build-a", "success"), + job(421, 42, "build-b", "success"), + job(422, 42, "qualify-candidate", "success"), + job(423, 42, "independent-release-policy", "success"), + job(424, 42, "publish-qualified-artifact", "skipped"), + ]; + const publishJobs = [ + job(430, 43, "publish-qualified-artifact", "failure", steps), + job(431, 43, "build-${{ matrix.builder }}", "skipped"), + job(432, 43, "qualify-candidate", "skipped"), + job(433, 43, "independent-release-policy", "skipped"), + ]; + const recoveryRun = { + id: "44", + attempt: 1, + url: `https://github.com/${repository}/actions/runs/44`, + headCommit: controllerCommit, + workflowPath: ".github/workflows/release-recovery.yml", + ref: "refs/heads/main", + }; + const jobsByRun: Record< + string, + { total_count: number; jobs: ReturnType[] }[] + > = { + "42": [{ total_count: candidateJobs.length, jobs: candidateJobs }], + "43": [{ total_count: publishJobs.length, jobs: publishJobs }], + }; + const releasesPages: { id: number; tag_name: string; draft: boolean }[][] = [ + [], + ]; + const input = { + tag, + candidateId: "42", + publishId: "43", + recoveryRun, + workflow: { + id: 346766046, + path: ".github/workflows/release.yml", + state: "active", + }, + runsPages: [ + { + total_count: 2, + workflow_runs: [run(42, "success"), run(43, "failure")], + }, + ], + jobsByRun, + releasesPages, + observedAt: "2026-09-19T21:00:00Z", + }; + const identity = { + tag, + tagCommit: sourceCommit, + mainTree: "c".repeat(40), + reviewedCandidateTree: "c".repeat(40), + packageName: "@davidahmann/mill", + version: "0.7.1", + }; + const artifact = Buffer.from("the already published artifact"); + const metadata = { + package: { name: "@davidahmann/mill", version: "0.7.1" }, + selectedArtifact: { + sha256: digest(artifact), + npmIntegrity: `sha512-${createHash("sha512").update(artifact).digest("base64")}`, + }, + }; + const log = [ + '##[group]Run gh api "repos/$GITHUB_REPOSITORY/actions/runs/$CANDIDATE_RUN_ID" > "$RUNNER_TEMP/candidate-run.json"', + "env:", + " CANDIDATE_RUN_ID: 42", + "##[endgroup]", + ...["genesis-candidate", "independent-policy"].flatMap((name) => [ + "##[group]Run actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c", + "with:", + ` name: ${name}-${tag}`, + ` repository: ${repository}`, + " run-id: 42", + "##[endgroup]", + ]), + ] + .map((line) => `2026-09-19T20:00:00.1234567Z ${line}`) + .join("\n"); + return { input, identity, metadata, artifact, log }; +} +type Fixture = ReturnType; +const candidate = (f: Fixture) => + required(required(f.input.runsPages[0]).workflow_runs[0]); +const publish = (f: Fixture) => + required(required(f.input.runsPages[0]).workflow_runs[1]); +const publishJobs = (f: Fixture) => + required(required(f.input.jobsByRun["43"])[0]); +const publishJob = (f: Fixture) => required(publishJobs(f).jobs[0]); +const publishStep = (f: Fixture, number: number) => + required(publishJob(f).steps.find((step) => step.number === number)); +const invoke = (args: string[], extraEnv: NodeJS.ProcessEnv = {}) => + spawnSync(process.execPath, [script, ...args], { + encoding: "utf8", + timeout: 10_000, + env: { ...env, ...extraEnv }, + }); +async function prepare(directory: string, f: Fixture) { + await Promise.all([ + writeFile( + path.join(directory, "recovery-observations.json"), + JSON.stringify(f.input), + ), + writeFile( + path.join(directory, "identity.json"), + JSON.stringify(f.identity), + ), + writeFile( + path.join(directory, "metadata.json"), + JSON.stringify(f.metadata), + ), + writeFile(path.join(directory, "publish-job.log"), f.log), + ]); + return [ + "validate", + directory, + path.join(directory, "identity.json"), + path.join(directory, "metadata.json"), + ]; +} + +describe("bounded finalization after immutable npm publication", () => { + it("retains the original publication identity and separately binds the main controller", async () => { + const temporary = await temporaryDirectory("mill-release-recovery-"); + try { + const f = fixture(); + const result = invoke(await prepare(temporary.path, f)); + expect(result.status, result.stderr).toBe(0); + const workflowRuns: unknown = JSON.parse( + await readFile(path.join(temporary.path, "workflow-runs.json"), "utf8"), + ); + expect(workflowRuns).toEqual({ + candidate: { + id: "42", + url: candidate(f).html_url, + headCommit: sourceCommit, + }, + publish: { + id: "43", + url: publish(f).html_url, + headCommit: sourceCommit, + }, + }); + const source: unknown = JSON.parse( + await readFile( + path.join(temporary.path, "recovery-source.json"), + "utf8", + ), + ); + expect(source).toMatchObject({ + source: { tagCommit: sourceCommit }, + recoveryRun: { id: "44", headCommit: controllerCommit }, + originalPublish: { + attempt: 1, + step: { + name: "Publish the exact preserved artifact with provenance", + conclusion: "success", + }, + candidateBinding: { candidateId: "42", logsDigest: digest(f.log) }, + }, + }); + const evidence = { + state: "verified", + package: { name: f.metadata.package.name, version: "0.7.1", tag }, + source: { + tagCommit: sourceCommit, + resultingMainCommit: sourceCommit, + resultingMainTree: f.identity.mainTree, + reviewedCandidateTree: f.identity.mainTree, + }, + selectedArtifact: f.metadata.selectedArtifact, + workflowRuns, + githubRelease: { state: "published", releaseId: "700" }, + }; + const evidenceFile = path.join( + temporary.path, + "release-evidence-final.json", + ); + const qualificationFile = path.join(temporary.path, "qualification.json"); + await writeFile(evidenceFile, JSON.stringify(evidence)); + await writeFile(qualificationFile, '{"historical":"qualification"}\n'); + const finalized = invoke([ + "finalize", + temporary.path, + evidenceFile, + qualificationFile, + ]); + expect(finalized.status, finalized.stderr).toBe(0); + expect( + JSON.parse( + await readFile( + path.join(temporary.path, "release-recovery.json"), + "utf8", + ), + ), + ).toMatchObject({ + releaseId: "700", + finalEvidenceFileDigest: digest(JSON.stringify(evidence)), + qualificationFileDigest: digest('{"historical":"qualification"}\n'), + workflowRuns, + }); + } finally { + await temporary.cleanup(); + } + }); + + const invalid: [string, (f: Fixture) => void][] = [ + [ + "foreign repository", + (f) => { + publish(f).repository.full_name = "foreign/mill"; + }, + ], + [ + "wrong workflow", + (f) => { + publish(f).workflow_id = 9; + }, + ], + [ + "wrong event", + (f) => { + publish(f).event = "push"; + }, + ], + [ + "wrong tag", + (f) => { + publish(f).head_branch = "v0.7.0"; + }, + ], + [ + "wrong source", + (f) => { + candidate(f).head_sha = "d".repeat(40); + }, + ], + [ + "unfinished run", + (f) => { + publish(f).status = "in_progress"; + }, + ], + [ + "repeated publish attempt", + (f) => { + publish(f).run_attempt = 2; + }, + ], + [ + "repeated candidate attempt", + (f) => { + candidate(f).run_attempt = 2; + }, + ], + [ + "wrong job attempt", + (f) => { + publishJob(f).run_attempt = 2; + }, + ], + [ + "job from another run", + (f) => { + publishJob(f).run_id = 90; + }, + ], + [ + "missing job page", + (f) => { + publishJobs(f).total_count++; + }, + ], + [ + "candidate failure", + (f) => { + candidate(f).conclusion = "failure"; + }, + ], + [ + "failed npm publication", + (f) => { + publishStep(f, 16).conclusion = "failure"; + }, + ], + [ + "unfinished npm publication", + (f) => { + publishStep(f, 16).status = "in_progress"; + }, + ], + [ + "failed prior qualification", + (f) => { + publishStep(f, 13).conclusion = "failure"; + }, + ], + [ + "missing readback failure", + (f) => { + publishStep(f, 17).conclusion = "success"; + }, + ], + [ + "GitHub create already attempted", + (f) => { + publishStep(f, 19).conclusion = "failure"; + }, + ], + [ + "duplicate npm step", + (f) => { + publishJob(f).steps.push({ ...publishStep(f, 16), number: 23 }); + }, + ], + [ + "unknown mutation step", + (f) => { + publishJob(f).steps.push({ + ...publishStep(f, 16), + number: 23, + name: "Another publication", + }); + }, + ], + [ + "existing draft on second page", + (f) => { + f.input.releasesPages.push([{ id: 90, tag_name: tag, draft: true }]); + }, + ], + [ + "existing public release", + (f) => { + f.input.releasesPages.push([{ id: 90, tag_name: tag, draft: false }]); + }, + ], + [ + "mismatched tag identity", + (f) => { + f.identity.tagCommit = controllerCommit; + }, + ], + [ + "different candidate consumed", + (f) => { + f.log = f.log.replace("CANDIDATE_RUN_ID: 42", "CANDIDATE_RUN_ID: 41"); + }, + ], + [ + "different candidate downloaded", + (f) => { + f.log = f.log.replace("run-id: 42", "run-id: 41"); + }, + ], + [ + "missing publish log binding", + (f) => { + f.log = ""; + }, + ], + [ + "duplicate candidate log binding", + (f) => { + f.log += `\n${f.log}`; + }, + ], + [ + "another publication run", + (f) => { + const run = { + ...publish(f), + id: 45, + html_url: `https://github.com/${repository}/actions/runs/45`, + }; + required(f.input.runsPages[0]).workflow_runs.push(run); + required(f.input.runsPages[0]).total_count++; + f.input.jobsByRun["45"] = [ + { total_count: 1, jobs: [{ ...publishJob(f), id: 450, run_id: 45 }] }, + ]; + }, + ], + ]; + it.each(invalid)( + "rejects %s before writing authority", + async (_name, mutate) => { + const temporary = await temporaryDirectory( + "mill-release-recovery-denial-", + ); + try { + const f = fixture(); + mutate(f); + expect(invoke(await prepare(temporary.path, f)).status).toBe(1); + await expect( + readFile(path.join(temporary.path, "workflow-runs.json")), + ).rejects.toThrow(); + } finally { + await temporary.cleanup(); + } + }, + ); + + it.each([ + { GITHUB_REF: "refs/tags/v0.7.1" }, + { GITHUB_RUN_ATTEMPT: "2" }, + { GITHUB_WORKFLOW_SHA: sourceCommit }, + { + GITHUB_WORKFLOW_REF: `${repository}/.github/workflows/other.yml@refs/heads/main`, + }, + ])("rejects changed controller context %j", async (extraEnv) => { + const temporary = await temporaryDirectory( + "mill-release-recovery-context-", + ); + try { + expect( + invoke(await prepare(temporary.path, fixture()), extraEnv).status, + ).toBe(1); + } finally { + await temporary.cleanup(); + } + }); + + it("checks the fresh complete release inventory and exact registry bytes", async () => { + const temporary = await temporaryDirectory("mill-release-recovery-bytes-"); + try { + const f = fixture(); + await prepare(temporary.path, f); + const inventory = path.join(temporary.path, "releases.json"); + await writeFile( + inventory, + JSON.stringify([[], [{ id: 7, tag_name: tag, draft: true }]]), + ); + expect(invoke(["absent", inventory, tag]).status).toBe(1); + await writeFile( + inventory, + JSON.stringify([[{ id: 6, tag_name: "v0.7.0", draft: false }]]), + ); + expect(invoke(["absent", inventory, tag]).status).toBe(0); + const registry = path.join(temporary.path, "registry"); + await mkdir(registry); + const artifact = path.join(registry, "mill.tgz"); + await writeFile(artifact, f.artifact); + expect( + invoke([ + "registry", + path.join(temporary.path, "metadata.json"), + registry, + ]).status, + ).toBe(0); + await writeFile(artifact, "different registry package"); + expect( + invoke([ + "registry", + path.join(temporary.path, "metadata.json"), + registry, + ]).status, + ).toBe(1); + } finally { + await temporary.cleanup(); + } + }); + + it("retains only necessary provider observations and keeps raw job logs private", async () => { + const temporary = await temporaryDirectory( + "mill-release-recovery-observe-", + ); + try { + const f = fixture(); + const root = `repos/${repository}/`; + const privateValue = "PRIVATE_UNNEEDED_PROVIDER_FIELD"; + const responses: Record = { + [`${root}actions/workflows/release.yml`]: { + ...f.input.workflow, + url: privateValue, + }, + [`${root}actions/workflows/release.yml/runs?event=workflow_dispatch&branch=${tag}&per_page=100`]: + f.input.runsPages.map((page) => ({ + ...page, + workflow_runs: page.workflow_runs.map((run) => ({ + ...run, + actor: { email: privateValue }, + head_commit: { message: privateValue }, + })), + })), + [`${root}releases?per_page=100`]: [ + [ + { + id: 1, + tag_name: "v0.7.0", + draft: false, + body: privateValue, + author: { email: privateValue }, + }, + ], + ], + [`${root}actions/jobs/430/logs`]: `${f.log}\n${privateValue}\n`, + }; + for (const [id, pages] of Object.entries(f.input.jobsByRun)) + responses[`${root}actions/runs/${id}/attempts/1/jobs?per_page=100`] = + pages.map((page) => ({ + ...page, + jobs: page.jobs.map((job) => ({ + ...job, + runner_name: privateValue, + })), + })); + const responsesFile = path.join(temporary.path, "api.json"); + await writeFile(responsesFile, JSON.stringify(responses)); + await writeFile( + path.join(temporary.path, "gh"), + `#!/usr/bin/env node +const fs=require("node:fs"); +const values=JSON.parse(fs.readFileSync(process.env.TEST_API_FILE,"utf8")); +const value=values[process.argv.at(-1)]; +if(value===undefined)process.exit(9); +process.stdout.write(typeof value==="string"?value:JSON.stringify(value)); +`, + { mode: 0o755 }, + ); + const result = invoke(["observe", tag, "42", "43", temporary.path], { + PATH: `${temporary.path}${path.delimiter}${process.env.PATH ?? ""}`, + TEST_API_FILE: responsesFile, + }); + expect(result.status, result.stderr).toBe(0); + for (const name of [ + "recovery-observations.json", + "candidate-run.json", + "candidate-jobs.json", + "publish-run.json", + ]) + expect( + await readFile(path.join(temporary.path, name), "utf8"), + ).not.toContain(privateValue); + expect( + await readFile(path.join(temporary.path, "publish-job.log"), "utf8"), + ).toContain(privateValue); + await writeFile( + path.join(temporary.path, "identity.json"), + JSON.stringify(f.identity), + ); + await writeFile( + path.join(temporary.path, "metadata.json"), + JSON.stringify(f.metadata), + ); + const validated = invoke([ + "validate", + temporary.path, + path.join(temporary.path, "identity.json"), + path.join(temporary.path, "metadata.json"), + ]); + expect(validated.status, validated.stderr).toBe(0); + expect( + await readFile( + path.join(temporary.path, "recovery-source.json"), + "utf8", + ), + ).not.toContain(privateValue); + } finally { + await temporary.cleanup(); + } + }); + + it("rejects changed final source, artifact and original workflow identities", async () => { + const temporary = await temporaryDirectory( + "mill-release-recovery-final-denial-", + ); + try { + const f = fixture(); + const validated = invoke(await prepare(temporary.path, f)); + expect(validated.status, validated.stderr).toBe(0); + const workflows: unknown = JSON.parse( + await readFile(path.join(temporary.path, "workflow-runs.json"), "utf8"), + ); + const evidence = { + state: "verified", + package: { tag }, + source: { + tagCommit: sourceCommit, + resultingMainCommit: sourceCommit, + resultingMainTree: f.identity.mainTree, + reviewedCandidateTree: f.identity.mainTree, + }, + selectedArtifact: f.metadata.selectedArtifact, + workflowRuns: workflows, + githubRelease: { state: "published", releaseId: "700" }, + }; + const file = path.join(temporary.path, "final.json"); + const qualification = path.join(temporary.path, "qualification.json"); + await writeFile(qualification, "{}"); + for (const mutation of [ + { source: { ...evidence.source, tagCommit: controllerCommit } }, + { + selectedArtifact: { + ...f.metadata.selectedArtifact, + sha256: digest("different"), + }, + }, + { + workflowRuns: { publish: { id: "44", headCommit: controllerCommit } }, + }, + { githubRelease: { state: "draft", releaseId: "700" } }, + ]) { + await writeFile(file, JSON.stringify({ ...evidence, ...mutation })); + expect( + invoke(["finalize", temporary.path, file, qualification]).status, + ).toBe(1); + } + await expect( + readFile(path.join(temporary.path, "release-recovery.json")), + ).rejects.toThrow(); + } finally { + await temporary.cleanup(); + } + }); + + it("protects recovery while keeping publication and model credentials out of package checks", async () => { + const text = await readFile( + ".github/workflows/release-recovery.yml", + "utf8", + ); + const workflow = parse(text) as { + on: Record; + concurrency: unknown; + permissions: Record; + jobs: { + finalize: { + if: string; + environment: string; + permissions: Record; + env: Record; + steps: { + name: string; + run?: string; + if?: unknown; + "continue-on-error"?: unknown; + uses?: string; + env?: Record; + with?: Record; + }[]; + }; + }; + }; + const job = workflow.jobs.finalize; + expect(Object.keys(workflow.on)).toEqual(["workflow_dispatch"]); + expect(workflow.concurrency).toEqual({ + group: "release-${{ inputs.tag }}", + "cancel-in-progress": false, + }); + expect(job.environment).toBe("npm"); + expect(job.if).toBe( + "github.ref == 'refs/heads/main' && github.repository == 'davidahmann/mill'", + ); + expect(job.permissions).toEqual({ actions: "read", contents: "write" }); + expect(workflow.permissions).toEqual({ actions: "read", contents: "read" }); + expect(job.env).not.toHaveProperty("GH_TOKEN"); + expect(text).not.toMatch(/npm publish|npm dist-tag|id-token:|--clobber/u); + const steps = job.steps; + const index = (name: string) => + steps.findIndex((step) => step.name === name); + const create = index("Create draft GitHub Release with exact artifacts"); + expect(create).toBeGreaterThan( + index("Read back npm and verify signatures"), + ); + expect(create).toBeGreaterThan( + index("Requalify the registry-downloaded package"), + ); + const createSource = required(steps[create]).run ?? ""; + expect( + createSource.indexOf( + ' absent "$RUNNER_TEMP/releases-before-create.json"', + ), + ).toBeGreaterThanOrEqual(0); + expect( + createSource.indexOf( + ' absent "$RUNNER_TEMP/releases-before-create.json"', + ), + ).toBeLessThan(createSource.indexOf("gh release create")); + expect(createSource).toContain("--paginate --slurp"); + expect( + required(steps[index("Requalify the registry-downloaded package")]).run, + ).toContain("--full-canary"); + for (const name of [ + "Read back npm and verify signatures", + "Requalify the registry-downloaded package", + "Install tagged tools without lifecycle scripts", + ]) + expect(required(steps[index(name)]).env).toBeUndefined(); + for (const step of steps) { + expect(step.if).toBeUndefined(); + expect(step["continue-on-error"]).toBeUndefined(); + } + expect( + required(steps[index("Checkout approved recovery controller")]).with, + ).toMatchObject({ + ref: "${{ github.sha }}", + path: "controller", + "persist-credentials": false, + }); + expect( + required(steps[index("Checkout immutable release source separately")]) + .with, + ).toMatchObject({ + ref: "${{ inputs.tag }}", + path: "release-source", + "persist-credentials": false, + }); + expect(text.match(/GITHUB_SHA="\$tag_commit"/gu)).toHaveLength(1); + expect(text.match(/gh release create/gu)).toHaveLength(1); + expect(text).toContain('cmp "$RUNNER_TEMP/release-recovery.json"'); + }); +});