From e174f639b329fe4986d58ec181ba7146549f1da2 Mon Sep 17 00:00:00 2001 From: David Ahmann <46606159+davidahmann@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:42:50 -0400 Subject: [PATCH 1/7] fix: recover verification of unchanged candidates Signed-off-by: David Ahmann <46606159+davidahmann@users.noreply.github.com> --- AGENTS.md | 10 +- CHANGELOG.md | 13 ++ README.md | 11 +- architecture/ARCHITECTURE.md | 15 +- docs/development.md | 8 + docs/releases/v0.8.1.md | 14 ++ docs/verifier-recovery.md | 51 ++++++ package-lock.json | 4 +- package.json | 2 +- product/verifier-recovery.md | 44 +++++ scripts/test-package.mjs | 148 +++++++++++++++- src/cli-program.ts | 58 ++++++- src/runtime/lifecycle.ts | 198 ++++++++++++++++++++-- src/runtime/state.ts | 76 +++++++++ src/runtime/timeline.ts | 4 + src/runtime/verification-recovery.ts | 145 ++++++++++++++++ src/version.ts | 2 +- test/runtime-lifecycle.test.ts | 244 ++++++++++++++++++++++++++- test/verification-recovery.test.ts | 132 +++++++++++++++ 19 files changed, 1150 insertions(+), 29 deletions(-) create mode 100644 docs/releases/v0.8.1.md create mode 100644 docs/verifier-recovery.md create mode 100644 product/verifier-recovery.md create mode 100644 src/runtime/verification-recovery.ts create mode 100644 test/verification-recovery.test.ts diff --git a/AGENTS.md b/AGENTS.md index 5ee3119..d8e5b0b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,6 +1,6 @@ # AGENTS.md: operating Mill safely -Version: 2.6 +Version: 2.7 Status: normative @@ -21,6 +21,10 @@ and checks are in `product/review-delivery-follow-through.md`. Use the native maintainer path for these Mill control changes; it does not grant a builder permission to modify its own frozen controls. +The owner approved the bounded verifier recovery repair on 2026-09-21. Its +authority and acceptance are in `product/verifier-recovery.md`. Use the native +maintainer route; preserve downstream task and candidate identities. + ## Operating Mill Mill turns approved repository-native product intent into a bounded, tested, @@ -259,6 +263,10 @@ millctl --json state backup millctl --json support-bundle --run ``` +- Eligible verifier infrastructure failures may use the exact attended + [candidate recovery](docs/verifier-recovery.md) plan. Its single-use allowance + retains the original deadline and grants only unchanged-candidate verification + and review. It cannot authorize another builder or repair. - `resume` is permitted only when Mill can prove no prior worker still owns the effect or when it is performing the approved bounded review-repair pass. - `cancel` records intent; only the live foreground controller may signal its diff --git a/CHANGELOG.md b/CHANGELOG.md index cc76dda..03c7256 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,19 @@ All notable changes follow Keep a Changelog and Semantic Versioning. ## [Unreleased] +## [0.8.1] - 2026-09-21 + +### Fixed + +- Recover unchanged candidates after eligible verifier infrastructure failures + through an exact attended plan, without another builder attempt. +- Preserve the original blocker when a follow-up command is rejected. + +### Added + +- A single bounded verification/review recovery allowance, with its own expiry + and controller identity while retaining the original run history. + ## [0.8.0] - 2026-09-21 ### Added diff --git a/README.md b/README.md index 70d06f7..d14e03b 100644 --- a/README.md +++ b/README.md @@ -158,14 +158,14 @@ existing repository. ```sh npm view @davidahmann/mill dist-tags --json -npm install --save-dev --save-exact --ignore-scripts @davidahmann/mill@0.8.0 +npm install --save-dev --save-exact --ignore-scripts @davidahmann/mill@0.8.1 npx --no-install millctl --version ``` -The example selects this source candidate's version, `0.8.0`. Use it after the +The example selects this source candidate's version, `0.8.1`. Use it after the linked release's final evidence confirms publication; an unpublished source candidate is not an installable release. See the -[v0.8.0 scope](docs/releases/v0.8.0.md) and +[v0.8.1 scope](docs/releases/v0.8.1.md) and [planning evidence rules](docs/planning.md#what-passing-evidence-means). To develop Mill itself from a clean source checkout: @@ -550,6 +550,11 @@ Cancellation records intent without discarding the receipt. Use `pr finalize` and green main checks before cleanup. See [approval recovery](docs/approvals.md#interruptions). +For an eligible pre-command verifier infrastructure failure, inspect the +[candidate recovery procedure](docs/verifier-recovery.md). It preserves the +original run and can grant one separately approved verification/review window +for the unchanged candidate. Tests and independent review remain required. + Before any remote attempt, stale review scope can use `review --task product/tasks/TASK.yaml --run --refresh --base --attended`. This preserves the candidate and remaining review budget, invalidates the diff --git a/architecture/ARCHITECTURE.md b/architecture/ARCHITECTURE.md index b95b8be..93420d4 100644 --- a/architecture/ARCHITECTURE.md +++ b/architecture/ARCHITECTURE.md @@ -324,11 +324,16 @@ private labels and binds removal readback to the observed container and daemon. Unresolved ownership blocks retry and destructive cleanup. There is no host-independent deadline watchdog; see [OCI recovery](../docs/oci-recovery.md). The persisted absolute run deadline is reused for verification, review, retry, -repair, and resume; no checkpoint grants a fresh budget. An attempt ID plus PID, -PGID, and process-start digest is diagnostic state, not signalling authority. -Cancellation is durable state polled by the foreground lease owner, which -terminates its own in-memory child, including a GitHub mutation process; no -command signals a stored PID. Cancellation is rechecked before each external +repair, and resume; ordinary checkpoints grant no fresh budget. The explicit +[candidate recovery](../docs/verifier-recovery.md) boundary may grant one +separately approved verification/review window after eligible infrastructure +failure. Its immutable receipt retains the original deadline, failure history, +controller identity and exact candidate. It grants no builder or repair and +requires settled worker/container ownership before application. An attempt ID +plus PID, PGID, and process-start digest is diagnostic state, not signalling +authority. Cancellation is durable state polled by the foreground lease owner, +which terminates its own in-memory child, including a GitHub mutation process; +no command signals a stored PID. Cancellation is rechecked before each external effect, and an interrupted effect remains unknown until authoritative readback. If the lease is free but a recorded process may still exist, resume and terminal cancellation fail closed for attended reconciliation. State events are diff --git a/docs/development.md b/docs/development.md index 16e3249..3fb363b 100644 --- a/docs/development.md +++ b/docs/development.md @@ -357,3 +357,11 @@ Native maintainer changes use the [standalone review route](maintainer-review.md). Dependabot grouping is checked by `npm run check:dependabot`; the checker covers Mill's bounded update policy and the shipped starter, not GitHub's full schema. + +## Verifier infrastructure recovery + +See [candidate recovery](verifier-recovery.md) for the exact plan/apply flow. +Keep preflight failures separate from failed command evidence. Recovery tests +must cover unchanged-candidate success, historical blocker provenance, pin +compatibility, stale approval, cancellation, uncertain ownership, authority +drift, original-deadline retention and denial of another builder or allowance. diff --git a/docs/releases/v0.8.1.md b/docs/releases/v0.8.1.md new file mode 100644 index 0000000..3d39b4e --- /dev/null +++ b/docs/releases/v0.8.1.md @@ -0,0 +1,14 @@ +# Mill 0.8.1 + +This source candidate adds explicit recovery for unchanged candidates blocked by +eligible verifier infrastructure failures. The operator approves one fixed +verification/review window; Mill preserves the original run, deadline, attempts, +candidate and failure history. Recovery grants no builder or repair. + +Rejected follow-up commands preserve the original blocker. Recovery checks +candidate and authority identity, previous evidence, worker and container +ownership, and its single-use approval before continuing. Tests remain required. + +Publication and npm/GitHub channel pointers are established by the protected +workflow's final release evidence. This source record grants no new stack, +customer-compatibility or unattended-operation claim. diff --git a/docs/verifier-recovery.md b/docs/verifier-recovery.md new file mode 100644 index 0000000..c319001 --- /dev/null +++ b/docs/verifier-recovery.md @@ -0,0 +1,51 @@ +# Recover verification of an unchanged candidate + +A missing approved image can stop verification before any test runs. Pull that +exact digest explicitly and inspect the run's status and timeline. Mill never +pulls it during verification, and restoring it does not establish passing tests. + +Recovery is available only for a committed candidate with an eligible +infrastructure failure. The task, configuration, frozen context, base, candidate +commit and tree must still match. Failed tests, changed files, cancellation, +uncertain workers or containers, and delivery effects require their existing +reconciliation or disposition. + +## One attended recovery window + +The recovery plan retains the original deadline and grants one separate window +for verification and read-only review of the same candidate. Select an explicit +future ISO timestamp no more than the smaller of the task's original duration +and 1,200 seconds away. Use the same timestamp in both commands: + +```sh +millctl --json verification-recovery plan --task product/tasks/TASK.yaml \ + --run --expires-at +millctl --json verification-recovery apply --task product/tasks/TASK.yaml \ + --run --expires-at \ + --approve sha256: --attended +millctl --json verify --task product/tasks/TASK.yaml --run +millctl --json review --task product/tasks/TASK.yaml --run +``` + +Inspect the exact plan before applying it. The approval binds the recorded +failure history, authority, candidate and recovery controller. Cross-version +recovery, including a repository without a version pin, requires the original +deadline to have expired, so the older pinned controller cannot start another +worker. A newer controller can service only the bound recovery operations; it +does not silently upgrade the repository's pinned tool. Preserve its exact +installed artifact and release evidence. Repin the downstream repository through +a reviewed change after the old run closes. + +The recovery grants no builder, repair, new candidate or delivery approval. +P0/P1 findings stop a candidate-only allowance. Required test failures remain +failures. Another expired recovery window requires a new owner decision outside +this single-use mechanism; repeating the command cannot extend it. + +## Historical failures + +Older Mill versions could replace an infrastructure blocker with +`RUN_NOT_COMMITTED` after an invalid retry. Recovery examines the retained event +chain instead of trusting that latest code alone. It must prove the eligible +failure belongs to the current candidate and excludes intervening validation, +review, cancellation and external effects. Retain both original and rejected +command evidence. Never edit the database to manufacture eligibility. diff --git a/package-lock.json b/package-lock.json index f34a283..62a922c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@davidahmann/mill", - "version": "0.8.0", + "version": "0.8.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@davidahmann/mill", - "version": "0.8.0", + "version": "0.8.1", "bundleDependencies": [ "typescript" ], diff --git a/package.json b/package.json index 4bc8367..21bc35a 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@davidahmann/mill", - "version": "0.8.0", + "version": "0.8.1", "description": "Local-first software factory for new and existing codebases. Turns approved product intent into tested, reviewed PRs with repo-native evidence and explicit human approval for delivery and merge.", "license": "Apache-2.0", "author": "David Ahmann", diff --git a/product/verifier-recovery.md b/product/verifier-recovery.md new file mode 100644 index 0000000..e1a0026 --- /dev/null +++ b/product/verifier-recovery.md @@ -0,0 +1,44 @@ +# Recover an unchanged candidate after verifier preflight failure + +Status: approved by David Ahmann on 2026-09-21 in the maintainer work session. +Route: native maintainer implementation, independent architecture review and +exact-candidate local review before push. + +Telryn's admitted T-019 builder committed its candidate, but verification +stopped before command execution because the approved image was unavailable. +Restoring the image did not make the blocked candidate retryable. A rejected +second verify also replaced the original blocker with `RUN_NOT_COMMITTED`. +Preserved events and CLI evidence retain the original cause. + +Implement a bounded recovery for that failure family. A passing image check must +not count as passing a product test. Preserve the run, original deadline, +candidate, task, configuration, context, attempts and prior evidence. + +Acceptance: + +- A rejected follow-up command preserves the existing failure reason. +- Safe pre-command infrastructure failure can retry only with unchanged + authority and candidate identity, no active or uncertain worker, and settled + container ownership. Actual failed commands require their existing + disposition. +- An attended, exact plan may grant one new candidate-only verification/review + window. Its expiry is fixed before approval and no later than the smaller of + the original task duration and 1,200 seconds from planning/application. +- Original deadlines and counters remain intact. Recovery never grants a + builder, repair, new candidate, weaker tests, automatic image pull, or + delivery approval. Cancellation, failed tests, drift and unresolved effects + block it. +- Historical overwritten blocker codes qualify only when the retained event + history proves the allowed failure and excludes intervening evidence/effects. +- A tool upgrade is explicit. Any exception to the frozen package pin binds the + recovery controller version to this run and its verification/review only. + Cross-version or unpinned recovery requires the original deadline to expire, + preventing the old controller from launching a builder or repair. +- Unit/integration and installed CLI tests cover success and adversarial + boundaries. Full native checks, audit and independent reviews pass before + merge. Release follows the existing tag/artifact/OIDC/readback procedure. + +After qualification, use the released fix to recover the existing Telryn +candidate under a fresh explicit receipt. Verify and review it before ordinary +approved delivery. Preserve acceptance and historical records; repin Telryn +through a separate reviewed maintainer change after the product run closes. diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index 4339036..32580a5 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -660,7 +660,7 @@ const stateFile=new URL("./containers.json",import.meta.url); if(args[0]==="info"){console.log(JSON.stringify("package-fixture-daemon"));process.exit(0)} let containers=existsSync(stateFile)?JSON.parse(readFileSync(stateFile,"utf8")):[]; const save=()=>writeFileSync(stateFile,JSON.stringify(containers)); -if(args[0]==="image"&&args[1]==="inspect"){process.exit(0)} +if(args[0]==="image"&&args[1]==="inspect"){process.exit(existsSync(new URL("./image-unavailable",import.meta.url))?1:0)} if(args[0]==="container"&&args[1]==="inspect"){ const found=containers.find(value=>value.name==="/"+args.at(-1)||value.id===args.at(-1)); if(found){console.log(JSON.stringify(found));process.exit(0)} @@ -1152,6 +1152,152 @@ else process.exit(2); process.stdout.write( "package review-policy canary passed: opt-in P2 advisory, opt-in P1 block, legacy P2 block, provider receipt rejection (fixture adapters)\n", ); + // Exercise public recovery commands from the installed tarball. The fake + // daemon fails before command execution; no model or registry is contacted. + await rm(path.join(tools, "review-scenario.json"), { force: true }); + const recoveryFixture = path.join(temporary, "verification-recovery"); + command( + gitExecutable, + ["clone", "--quiet", "--no-hardlinks", consumer, recoveryFixture], + temporary, + ); + await writeFile( + path.join(recoveryFixture, "mill.lock"), + `schemaVersion: "1" +mill: + package: "@davidahmann/mill" + version: "${MILL_VERSION}" +`, + ); + command(gitExecutable, ["add", "mill.lock"], recoveryFixture); + command( + gitExecutable, + [ + "-c", + "user.name=Mill Package Test", + "-c", + "user.email=mill-package@example.invalid", + "commit", + "--no-gpg-sign", + "-m", + "test: pin candidate recovery controller", + ], + recoveryFixture, + ); + const recoveryEnvironment = { + ...canaryEnvironment, + MILL_STATE_HOME: path.join(state, "verification-recovery"), + }; + const recoveryCall = (args, succeeds = true) => { + const result = spawnSync( + bin, + ["--json", "--cwd", recoveryFixture, ...args], + { + cwd: recoveryFixture, + env: recoveryEnvironment, + encoding: "utf8", + timeout: 120_000, + }, + ); + if ((result.status === 0) !== succeeds) { + throw new Error( + `packed recovery command unexpected status: ${result.stdout}\n${result.stderr}`, + ); + } + return JSON.parse(result.stdout); + }; + const recoveryTask = "product/tasks/canary-1.yaml"; + const recoveryBase = recoveryCall([ + "qualify", + "--baseline", + "--task", + recoveryTask, + ]); + const recoveryStarted = recoveryCall([ + "run", + "--task", + recoveryTask, + "--approve", + recoveryBase.data.approvalDigest, + "--attended", + ]); + const recoveryRun = recoveryStarted.data.run; + const recoveryArgs = ["--task", recoveryTask, "--run", recoveryRun.id]; + await writeFile(path.join(tools, "image-unavailable"), "missing image\n"); + const infraFailure = recoveryCall(["verify", ...recoveryArgs], false); + if (infraFailure.reasons?.[0]?.code !== "VERIFIER_IMAGE_UNAVAILABLE") { + throw new Error( + "packed recovery did not preserve pre-command image failure", + ); + } + recoveryCall(["verify", ...recoveryArgs], false); + const stillBlocked = recoveryCall(["status", "--run", recoveryRun.id]).data + .run; + if (stillBlocked.blockCode !== "VERIFIER_IMAGE_UNAVAILABLE") { + throw new Error( + "rejected verify replaced the original infrastructure blocker", + ); + } + await rm(path.join(tools, "image-unavailable")); + const recoveryExpiry = new Date(Date.now() + 50_000).toISOString(); + const recoveryProposal = recoveryCall([ + "verification-recovery", + "plan", + ...recoveryArgs, + "--expires-at", + recoveryExpiry, + ]); + recoveryCall( + [ + "verification-recovery", + "apply", + ...recoveryArgs, + "--expires-at", + recoveryExpiry, + "--approve", + recoveryProposal.data.approvalDigest, + ], + false, + ); + recoveryCall([ + "verification-recovery", + "apply", + ...recoveryArgs, + "--expires-at", + recoveryExpiry, + "--approve", + recoveryProposal.data.approvalDigest, + "--attended", + ]); + recoveryCall(["verify", ...recoveryArgs]); + recoveryCall(["review", ...recoveryArgs]); + const recovered = recoveryCall(["status", "--run", recoveryRun.id]).data.run; + if ( + recovered.status !== "reviewed" || + recovered.candidateCommit !== recoveryRun.candidateCommit || + recovered.candidateTree !== recoveryRun.candidateTree || + recovered.deadlineAt !== recoveryRun.deadlineAt || + recovered.attemptCount !== 1 || + recovered.repairCount !== 0 + ) { + throw new Error( + "packed recovery changed original authority or failed exact review", + ); + } + recoveryCall( + [ + "verification-recovery", + "plan", + ...recoveryArgs, + "--expires-at", + recoveryExpiry, + ], + false, + ); + process.stdout.write( + "package candidate-recovery canary passed: preserved blocker, exact approval, attendance, unchanged candidate/deadline/attempts, verification and review, repeat denial (fixture adapters)\n", + ); + process.stdout.write( `package draft-PR lifecycle canary passed: ${packResult.filename}\n`, ); diff --git a/src/cli-program.ts b/src/cli-program.ts index f08cd18..e9057fd 100644 --- a/src/cli-program.ts +++ b/src/cli-program.ts @@ -65,6 +65,8 @@ import { stateRestore, supportBundle, verifyRun, + planVerificationRecovery, + recoverVerification, } from "./runtime/lifecycle.js"; import { builderIsolationBoundary, @@ -1671,6 +1673,58 @@ export function createProgram(io: CliIo, jsonErrors = false): Command { } }); + const verificationRecovery = program + .command("verification-recovery") + .description("plan or approve bounded recovery of one unchanged candidate"); + for (const mode of ["plan", "apply"] as const) { + verificationRecovery + .command(mode) + .requiredOption("--task ", "original approved task packet") + .requiredOption("--run ", "existing blocked run") + .requiredOption( + "--expires-at ", + "fixed ISO expiry for verification and review only", + ) + .option("--approve ", "exact recovery plan digest") + .option("--attended", "confirm attended recovery") + .action( + async (options: { + task: string; + run: string; + expiresAt: string; + approve?: string; + attended?: boolean; + }) => { + const global = globals(program); + const root = await findRepositoryRoot(global.cwd); + const input = { + root, + taskPath: options.task, + runId: options.run, + expiresAt: options.expiresAt, + }; + const result = + mode === "plan" + ? await planVerificationRecovery(input) + : await recoverVerification({ + ...input, + approvalDigest: options.approve ?? "", + attended: options.attended === true, + }); + emit( + io, + global.json === true, + commandResult({ + command: `verification-recovery.${mode}`, + ok: true, + status: "ok", + data: result, + }), + ); + }, + ); + } + program .command("verify") .description( @@ -1681,7 +1735,7 @@ export function createProgram(io: CliIo, jsonErrors = false): Command { .action(async (options: { task: string; run: string }) => { const global = globals(program); const root = await findRepositoryRoot(global.cwd); - await enforceExactVersion(root); + const result = await verifyRun({ root, taskPath: options.task, @@ -1738,7 +1792,7 @@ export function createProgram(io: CliIo, jsonErrors = false): Command { }) => { const global = globals(program); const root = await findRepositoryRoot(global.cwd); - await enforceExactVersion(root); + const result = await reviewRun({ root, taskPath: options.task, diff --git a/src/runtime/lifecycle.ts b/src/runtime/lifecycle.ts index 29a562c..7a84b64 100644 --- a/src/runtime/lifecycle.ts +++ b/src/runtime/lifecycle.ts @@ -1,3 +1,13 @@ +import { readLockStatus, enforceExactVersion } from "../config/lock.js"; +import { safeReadText } from "../security/safe-path.js"; +import { + verificationRecoverySchema, + eligibleVerificationFailure, + recoveryCheckpoint, + recoveryDigest, + recoveryError, + type VerificationRecovery, +} from "./verification-recovery.js"; import { blockingReviewFindings, classifyReview } from "./review-policy.js"; import { createHash, randomUUID } from "node:crypto"; import { lstat, readFile } from "node:fs/promises"; @@ -444,7 +454,11 @@ async function admitWorker(input: { ? {} : { impactManifestDigest: input.inputs.continuity.impactDigest }), profile, - deadlineAt: input.run.deadlineAt, + deadlineAt: + input.role === "reviewer" + ? (input.store.verificationRecovery(input.run.id)?.expiresAt ?? + input.run.deadlineAt) + : input.run.deadlineAt, }); input.store.admitWorkerInvocation({ runId: input.run.id, @@ -880,6 +894,159 @@ export async function qualifyBaseline(input: { } } +async function recoveryLock( + root: string, +): Promise<{ pinnedVersion: string | null; lockDigest: string | null }> { + const lock = await readLockStatus(root); + return { + pinnedVersion: lock.requiredVersion ?? null, + lockDigest: lock.found + ? canonicalDigest(await safeReadText(root, "mill.lock", 256 * 1024)) + : null, + }; +} + +async function verificationDeadline( + root: string, + store: StateStore, + run: RunRecord, +): Promise { + const receipt = store.verificationRecovery(run.id); + if (receipt === undefined) { + await enforceExactVersion(root); + return persistedRunDeadline(run); + } + const lock = await recoveryLock(root); + if ( + receipt.controllerVersion !== MILL_VERSION || + receipt.lockDigest !== lock.lockDigest || + receipt.pinnedVersion !== lock.pinnedVersion + ) + recoveryError("Recovery controller or original pin changed."); + if (Date.parse(receipt.expiresAt) <= Date.now()) + recoveryError("The fixed recovery allowance expired."); + return Date.parse(receipt.expiresAt); +} + +async function createVerificationRecoveryPlan( + root: string, + context: RunContext, + runId: string, + expiresAt: string, +): Promise { + const { store, inputs } = context; + const run = store.getRun(runId); + if (store.verificationRecovery(runId) !== undefined) + recoveryError("Recovery is single use."); + if ( + store.unresolvedMutatingWorkerInvocations(runId).length > 0 || + (await hasPendingOciResources(store.directory)) + ) + recoveryError("Worker or OCI ownership must be reconciled first."); + assertEffectAllowsNewWork(run); + const failureSequence = eligibleVerificationFailure(run, store.events(runId)); + const candidate = await assertRunBindings(root, run, inputs); + const lock = await recoveryLock(root); + if ( + lock.pinnedVersion !== MILL_VERSION && + Date.parse(run.deadlineAt) > Date.now() + ) + recoveryError( + "Cross-version recovery requires expiration of the original builder deadline.", + ); + const expiry = Date.parse(expiresAt); + if ( + !Number.isFinite(expiry) || + expiry <= Date.now() || + expiry > + Date.now() + Math.min(1200, inputs.task.budget.deadlineSeconds) * 1000 + ) + recoveryError( + "Supply a fixed future expiry within the task deadline and 1200-second maximum.", + ); + return verificationRecoverySchema.parse({ + schemaVersion: "1", + scope: "candidate_verification_review_only", + runId, + repositoryId: run.repositoryId, + taskDigest: run.taskDigest, + configDigest: run.configDigest, + baseCommit: run.baseCommit, + candidateCommit: candidate.commit, + candidateTree: candidate.tree, + contextDigest: run.contextDigest, + controlDigest: canonicalDigest(run.controlJson ?? null), + originalDeadlineAt: run.deadlineAt, + expiresAt, + checkpointDigest: recoveryCheckpoint(store.events(runId)), + failureSequence, + controllerVersion: MILL_VERSION, + ...lock, + builderAttempts: 0, + repairGenerations: 0, + }); +} + +export async function planVerificationRecovery(input: { + root: string; + taskPath: string; + runId: string; + expiresAt: string; +}): Promise<{ plan: VerificationRecovery; approvalDigest: string }> { + const context = await openRunContext(input.root, input.taskPath); + try { + const plan = await createVerificationRecoveryPlan( + input.root, + context, + input.runId, + input.expiresAt, + ); + return { plan, approvalDigest: recoveryDigest(plan) }; + } finally { + context.store.close(); + } +} + +export async function recoverVerification(input: { + root: string; + taskPath: string; + runId: string; + expiresAt: string; + approvalDigest: string; + attended: boolean; +}): Promise { + if (!input.attended) + throw new MillError( + "ATTENDANCE_REQUIRED", + "Candidate recovery requires attended exact-plan approval.", + ExitCode.configuration, + ); + const context = await openRunContext(input.root, input.taskPath); + let lease: Awaited> | undefined; + try { + lease = await acquireWriterLease(context.store); + const plan = await createVerificationRecoveryPlan( + input.root, + context, + input.runId, + input.expiresAt, + ); + return publicRunRecord( + context.store.applyVerificationRecovery( + input.runId, + plan, + input.approvalDigest, + ), + ); + } finally { + try { + await lease?.release(); + } finally { + context.store.close(); + } + } +} + export async function verifyRun(input: { root: string; taskPath: string; @@ -889,6 +1056,7 @@ export async function verifyRun(input: { const { inputs, store } = context; let lease: Awaited> | undefined; const signals = processCancellationScope(); + let operationStarted = false; try { lease = await acquireWriterLease(store); await reconcileOciResources({ @@ -905,7 +1073,7 @@ export async function verifyRun(input: { ExitCode.configuration, ); } - const deadlineMs = persistedRunDeadline(run); + const deadlineMs = await verificationDeadline(input.root, store, run); const candidate = await assertRunBindings(input.root, run, inputs); const hooks = lifecycleHooks(store, run.id); const dependencyRoot = await dependencySnapshotDirectory({ @@ -913,6 +1081,7 @@ export async function verifyRun(input: { stateDirectory: store.directory, config: inputs.config, }); + operationStarted = true; const evidence = await verifyDeclaredCommands({ stateDirectory: store.directory, root: candidate.worktree, @@ -956,7 +1125,8 @@ export async function verifyRun(input: { }; } catch (error) { const failure = asMillError(error); - if (lease !== undefined) settleFailure(store, input.runId, failure); + if (lease !== undefined && operationStarted) + settleFailure(store, input.runId, failure); throw failure; } finally { signals.dispose(); @@ -995,7 +1165,7 @@ export async function reviewRun(input: { reconcileMutatingWorkerAdmissions(store, run, storedActiveProcess(run)); run = store.getRun(run.id); assertEffectAllowsNewWork(run); - const deadlineMs = persistedRunDeadline(run); + const deadlineMs = await verificationDeadline(input.root, store, run); if (input.refresh === true) { if (input.attended !== true) throw new MillError( @@ -1050,15 +1220,15 @@ export async function reviewRun(input: { "WORKER_RESULT_MISSING", "WORKER_RESULT_CONFLICT", ]); - if ( + const retryReview = run.status === "blocked" && run.blockCode !== undefined && retryableReviewBlocks.has(run.blockCode) && - run.validationJson !== undefined + run.validationJson !== undefined; + if ( + (!retryReview && run.status !== "verified") || + run.validationJson === undefined ) { - run = store.transition(run.id, "verified", "review.retry_ready"); - } - if (run.status !== "verified" || run.validationJson === undefined) { throw new MillError( "RUN_NOT_VERIFIED", "Only an exact verified candidate can be reviewed.", @@ -1106,6 +1276,10 @@ export async function reviewRun(input: { "The prepared review scope no longer matches the candidate.", ExitCode.configuration, ); + if (retryReview) { + store.assertReviewBudget(run.id, inputs.task.budget.retryCount + 1); + run = store.transition(run.id, "verified", "review.retry_ready"); + } const reviewAttempt = store.beginReviewAttempt( run.id, inputs.task.budget.retryCount + 1, @@ -1167,7 +1341,7 @@ export async function reviewRun(input: { } } catch (error) { const failure = asMillError(error); - if (lease !== undefined && (input.refresh !== true || reviewPrepared)) + if (lease !== undefined && reviewPrepared) settleFailure(store, input.runId, failure); throw failure; } finally { @@ -1195,6 +1369,7 @@ export async function resumeRun(input: { stateDirectory: store.directory, root: input.root, }); + store.assertNoRecoveryBuilder(input.runId); let run = store.getRun(input.runId); const active = storedActiveProcess(run); reconcileMutatingWorkerAdmissions(store, run, active); @@ -1361,7 +1536,8 @@ export async function resumeRun(input: { return publicRunRecord(store.getRun(run.id)); } catch (error) { const failure = asMillError(error); - if (lease !== undefined) settleFailure(store, input.runId, failure); + if (lease !== undefined && store.getRun(input.runId).status === "running") + settleFailure(store, input.runId, failure); throw failure; } finally { signals.dispose(); diff --git a/src/runtime/state.ts b/src/runtime/state.ts index f7b8224..d0e26c5 100644 --- a/src/runtime/state.ts +++ b/src/runtime/state.ts @@ -1,3 +1,12 @@ +import { + verificationRecoverySchema, + assertRecoveryBinding, + recoveryCheckpoint, + recoveryDigest, + eligibleVerificationFailure, + recoveryError, + type VerificationRecovery, +} from "./verification-recovery.js"; import { blockingReviewFindings } from "./review-policy.js"; import { createHash, randomUUID } from "node:crypto"; import { constants } from "node:fs"; @@ -1123,7 +1132,69 @@ export class StateStore { return this.getRun(id); } + verificationRecovery(id: string): VerificationRecovery | undefined { + const events = this.events(id).filter( + (event) => event.type === "verification.recovery_applied", + ); + if (events.length === 0) return undefined; + if (events.length !== 1) + recoveryError("Multiple recovery receipts are invalid."); + const data = events[0]?.data as { + receipt?: string; + approvalDigest?: string; + }; + const plan = verificationRecoverySchema.parse( + JSON.parse(data.receipt ?? "null"), + ); + if (recoveryDigest(plan) !== data.approvalDigest) + recoveryError("Recovery receipt digest mismatch."); + assertRecoveryBinding(this.getRun(id), plan); + return plan; + } + + applyVerificationRecovery( + id: string, + plan: VerificationRecovery, + approvalDigest: string, + ): RunRecord { + this.#transaction(() => { + const run = this.getRun(id); + if (this.verificationRecovery(id) !== undefined) + recoveryError("Recovery is single use."); + assertRecoveryBinding(run, plan); + if ( + recoveryDigest(plan) !== approvalDigest || + recoveryCheckpoint(this.events(id)) !== plan.checkpointDigest || + eligibleVerificationFailure(run, this.events(id)) !== + plan.failureSequence || + Date.parse(plan.expiresAt) <= Date.now() + ) + recoveryError("Recovery approval is stale, expired or mismatched."); + assertEffectAllowsNewWork(run); + this.#database + .prepare( + "UPDATE runs SET status = 'committed', block_code = NULL, updated_at = ? WHERE id = ?", + ) + .run(new Date().toISOString(), id); + this.#event(id, "verification.recovery_applied", { + from: "blocked", + to: "committed", + receipt: JSON.stringify(plan), + approvalDigest, + }); + }); + return this.getRun(id); + } + + assertNoRecoveryBuilder(id: string): void { + if (this.verificationRecovery(id) !== undefined) + recoveryError( + "Candidate recovery grants no builder or repair authority.", + ); + } + beginRepair(id: string, maximumRepairGenerations = 1): RunRecord { + this.assertNoRecoveryBuilder(id); this.#transaction(() => { const current = this.getRun(id); if ( @@ -1286,6 +1357,10 @@ export class StateStore { return row.count + 1; } + assertReviewBudget(id: string, maximum: number): void { + this.#assertReviewBudget(this.getRun(id), maximum); + } + beginReviewAttempt(id: string, maximum: number): number { let attempt = 0; this.#transaction(() => { @@ -1401,6 +1476,7 @@ export class StateStore { } beginBuilderAttempt(id: string, maximum: number): void { + this.assertNoRecoveryBuilder(id); if (this.getRun(id).cancelRequested) { throw new MillError( "OPERATOR_CANCELLED", diff --git a/src/runtime/timeline.ts b/src/runtime/timeline.ts index b4eeb02..6eb425a 100644 --- a/src/runtime/timeline.ts +++ b/src/runtime/timeline.ts @@ -42,6 +42,7 @@ const lifecycleTransitionEventTypes = new Set([ "run.closed", "run.interrupted", "run.ready", + "verification.recovery_applied", "validation.failed", "validation.passed", "workspace.setup_failed", @@ -89,6 +90,9 @@ function isRecordedTransitionAllowed( ): boolean { return ( isRunTransitionAllowed(from, to) || + (type === "verification.recovery_applied" && + from === "blocked" && + to === "committed") || (type === "review.refresh_prepared" && ["reviewed", "proposing"].includes(from) && to === "verified") || diff --git a/src/runtime/verification-recovery.ts b/src/runtime/verification-recovery.ts new file mode 100644 index 0000000..1526e0d --- /dev/null +++ b/src/runtime/verification-recovery.ts @@ -0,0 +1,145 @@ +import { z } from "zod"; +import { canonicalDigest, type JsonValue } from "../contracts/canonical.js"; +import { ExitCode, MillError } from "../errors.js"; +import type { RunRecord } from "./state.js"; + +const digest = z.string().regex(/^sha256:[a-f0-9]{64}$/u); +const commit = z.string().regex(/^[a-f0-9]{40}$/u); +export const verificationRecoverySchema = z + .object({ + schemaVersion: z.literal("1"), + scope: z.literal("candidate_verification_review_only"), + runId: z.string(), + repositoryId: z.string(), + taskDigest: digest, + configDigest: digest, + baseCommit: commit, + candidateCommit: commit, + candidateTree: commit, + contextDigest: digest, + controlDigest: digest, + originalDeadlineAt: z.iso.datetime(), + expiresAt: z.iso.datetime(), + checkpointDigest: digest, + failureSequence: z.number().int().positive(), + controllerVersion: z.string(), + lockDigest: digest.nullable(), + pinnedVersion: z.string().nullable(), + builderAttempts: z.literal(0), + repairGenerations: z.literal(0), + }) + .strict(); +export type VerificationRecovery = z.infer; +export function recoveryError(message: string): never { + throw new MillError( + "VERIFICATION_RECOVERY_UNAVAILABLE", + message, + ExitCode.configuration, + ); +} +export function recoveryDigest(plan: VerificationRecovery): string { + return canonicalDigest(plan); +} +export function recoveryCheckpoint( + events: readonly Record[], +): string { + return canonicalDigest(events as unknown as JsonValue); +} +export function eligibleVerificationFailure( + run: RunRecord, + events: readonly Record[], +): number { + if ( + run.status !== "blocked" || + run.cancelRequested || + run.validationJson !== undefined || + run.reviewJson !== undefined || + run.deliveryJson !== undefined || + run.remoteFeedbackJson !== undefined || + run.activeProcessId !== undefined || + run.candidateCommit === undefined || + run.candidateTree === undefined + ) + recoveryError( + "Recovery requires a stopped, unvalidated committed candidate without delivery or cancellation.", + ); + const lastCommit = events.findLastIndex( + (event) => event.type === "candidate.committed", + ); + if (lastCommit < 0) + recoveryError("No lifecycle-owned candidate event exists."); + const committed = events[lastCommit]?.data as + Record | undefined; + if ( + committed?.commit !== run.candidateCommit || + committed.tree !== run.candidateTree + ) + recoveryError( + "The candidate event does not bind the current commit and tree.", + ); + let previous = 0; + for (const event of events) { + if ( + !Number.isSafeInteger(event.sequence) || + Number(event.sequence) <= previous + ) + recoveryError("The recovery journal sequence is invalid."); + previous = Number(event.sequence); + } + let failureSequence: number | undefined; + let blocker: string | undefined; + for (const event of events.slice(lastCommit + 1)) { + const data = event.data as Record | undefined; + if (event.type === "command.rejected") continue; + if (failureSequence === undefined && event.type === "builder.completed") + continue; + if ( + failureSequence === undefined && + event.type === "run.blocked" && + data?.from === "committed" && + data.to === "blocked" && + ["VERIFIER_IMAGE_UNAVAILABLE", "OCI_RUNTIME_UNAVAILABLE"].includes( + String(data.code), + ) + ) { + failureSequence = Number(event.sequence); + blocker = String(data.code); + continue; + } + if ( + failureSequence !== undefined && + event.type === "run.blocker_replaced" && + data?.from === blocker && + data?.to === "RUN_NOT_COMMITTED" + ) { + blocker = "RUN_NOT_COMMITTED"; + continue; + } + recoveryError( + "The journal contains work beyond an eligible verifier preflight failure.", + ); + } + if (failureSequence === undefined || blocker !== run.blockCode) + recoveryError( + "The current blocker is not proven by verifier infrastructure failure history.", + ); + return failureSequence; +} +export function assertRecoveryBinding( + run: RunRecord, + plan: VerificationRecovery, +): void { + if ( + plan.runId !== run.id || + plan.repositoryId !== run.repositoryId || + plan.taskDigest !== run.taskDigest || + plan.configDigest !== run.configDigest || + plan.baseCommit !== run.baseCommit || + plan.candidateCommit !== run.candidateCommit || + plan.candidateTree !== run.candidateTree || + plan.contextDigest !== run.contextDigest || + plan.originalDeadlineAt !== run.deadlineAt || + plan.controlDigest !== canonicalDigest(run.controlJson ?? null) + ) + recoveryError("Recovery receipt does not match the frozen run."); +} diff --git a/src/version.ts b/src/version.ts index 8e38362..02a557b 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,3 +1,3 @@ export const MILL_PACKAGE = "@davidahmann/mill"; -export const MILL_VERSION = "0.8.0"; +export const MILL_VERSION = "0.8.1"; export const RESULT_SCHEMA_VERSION = "1"; diff --git a/test/runtime-lifecycle.test.ts b/test/runtime-lifecycle.test.ts index cb709a8..0a99d39 100644 --- a/test/runtime-lifecycle.test.ts +++ b/test/runtime-lifecycle.test.ts @@ -1,6 +1,13 @@ import { execFile } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; -import { chmod, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import { + chmod, + mkdir, + readFile, + readdir, + rm, + writeFile, +} from "node:fs/promises"; import path from "node:path"; import { trackFakeDocker } from "./fake-oci.js"; import { DatabaseSync } from "node:sqlite"; @@ -19,6 +26,8 @@ import { statePurge, supportBundle, verifyRun, + planVerificationRecovery, + recoverVerification, } from "../src/runtime/lifecycle.js"; import { buildContextManifest } from "../src/runtime/context.js"; import { loadRuntimeInputs } from "../src/runtime/inputs.js"; @@ -86,6 +95,217 @@ async function qualifiedApproval( } describe("local delivery lifecycle", () => { + it("binds an expired original pin to the recovery controller without changing frozen files", async () => { + const fixture = await runtimeFixture(); + activate(fixture); + try { + await writeFile( + path.join(fixture.root, "mill.lock"), + 'schemaVersion: "1"\nmill:\n package: "@davidahmann/mill"\n version: "0.8.0"\n', + ); + await git(fixture.root, ["add", "mill.lock"]); + await git(fixture.root, [ + "commit", + "--no-gpg-sign", + "-m", + "test: historical pin", + ]); + const started = await startLocalRun({ + root: fixture.root, + taskPath: fixture.taskPath, + approvalDigest: await qualifiedApproval(fixture), + }); + const input = { + root: fixture.root, + taskPath: fixture.taskPath, + runId: started.run.id, + }; + const store = await StateStore.open( + "11111111-1111-4111-8111-111111111111", + await commonGitDirectory(fixture.root), + ); + store.transition(input.runId, "blocked", "run.blocked", { + code: "VERIFIER_IMAGE_UNAVAILABLE", + }); + const dbPath = store.databasePath; + store.close(); + const expiresAt = new Date(Date.now() + 55000).toISOString(); + await expect( + planVerificationRecovery({ ...input, expiresAt }), + ).rejects.toMatchObject({ code: "VERIFICATION_RECOVERY_UNAVAILABLE" }); + const db = new DatabaseSync(dbPath); + db.prepare("UPDATE runs SET deadline_at = ? WHERE id = ?").run( + "2020-01-01T00:00:00.000Z", + input.runId, + ); + db.close(); + const proposal = await planVerificationRecovery({ ...input, expiresAt }); + expect(proposal.plan.pinnedVersion).toBe("0.8.0"); + await recoverVerification({ + ...input, + expiresAt, + approvalDigest: proposal.approvalDigest, + attended: true, + }); + expect((await verifyRun(input)).evidence.passed).toBe(true); + expect((await reviewRun(input)).run.status).toBe("reviewed"); + expect( + await readFile(path.join(fixture.root, "mill.lock"), "utf8"), + ).toContain('"0.8.0"'); + } finally { + await fixture.cleanup(); + } + }); + it.each([false, true])( + "recovers only the unchanged infrastructure-blocked candidate, legacy=%s", + async (legacy) => { + const fixture = await runtimeFixture(); + activate(fixture); + try { + const started = await startLocalRun({ + root: fixture.root, + taskPath: fixture.taskPath, + approvalDigest: await qualifiedApproval(fixture), + }); + const input = { + root: fixture.root, + taskPath: fixture.taskPath, + runId: started.run.id, + }; + const docker = await readFile(fixture.dockerPath, "utf8"); + await writeFile( + fixture.dockerPath, + docker.replace( + "const args=process.argv.slice(2);", + 'const args=process.argv.slice(2);if(args[0]==="image")process.exit(1);', + ), + ); + await expect(verifyRun(input)).rejects.toMatchObject({ + code: "VERIFIER_IMAGE_UNAVAILABLE", + }); + await expect(verifyRun(input)).rejects.toMatchObject({ + code: "RUN_NOT_COMMITTED", + }); + await expect(reviewRun(input)).rejects.toMatchObject({ + code: "RUN_NOT_VERIFIED", + }); + expect((await runStatus(input)).run?.blockCode).toBe( + "VERIFIER_IMAGE_UNAVAILABLE", + ); + const store = await StateStore.open( + "11111111-1111-4111-8111-111111111111", + await commonGitDirectory(fixture.root), + ); + const dbPath = store.databasePath; + if (legacy) + store.replaceBlocker( + input.runId, + "RUN_NOT_COMMITTED", + "run.blocker_replaced", + ); + store.close(); + await expect( + planVerificationRecovery({ + ...input, + expiresAt: new Date(Date.now() + 55000).toISOString(), + }), + ).rejects.toMatchObject({ code: "VERIFICATION_RECOVERY_UNAVAILABLE" }); + const db = new DatabaseSync(dbPath); + db.prepare("UPDATE runs SET deadline_at = ? WHERE id = ?").run( + "2020-01-01T00:00:00.000Z", + input.runId, + ); + db.close(); + await expect(resumeRun(input)).rejects.toMatchObject({ + code: "RUN_DEADLINE_EXCEEDED", + }); + const expiresAt = new Date(Date.now() + 55000).toISOString(); + const proposal = await planVerificationRecovery({ + ...input, + expiresAt, + }); + await expect( + recoverVerification({ + ...input, + expiresAt, + approvalDigest: proposal.approvalDigest, + attended: false, + }), + ).rejects.toMatchObject({ code: "ATTENDANCE_REQUIRED" }); + await expect( + recoverVerification({ + ...input, + expiresAt, + approvalDigest: "sha256:" + "0".repeat(64), + attended: true, + }), + ).rejects.toMatchObject({ code: "VERIFICATION_RECOVERY_UNAVAILABLE" }); + const proofStore = await StateStore.open( + "11111111-1111-4111-8111-111111111111", + await commonGitDirectory(fixture.root), + ); + const pending = path.join( + proofStore.directory, + "oci-resources", + "bad.json", + ); + await mkdir(path.dirname(pending), { recursive: true }); + await writeFile(pending, "{}"); + await expect( + planVerificationRecovery({ ...input, expiresAt }), + ).rejects.toThrow(); + await rm(pending); + proofStore.recordEvent(input.runId, "command.rejected", { + code: "TEST_STALE_APPROVAL", + }); + proofStore.close(); + await expect( + recoverVerification({ + ...input, + expiresAt, + approvalDigest: proposal.approvalDigest, + attended: true, + }), + ).rejects.toMatchObject({ code: "VERIFICATION_RECOVERY_UNAVAILABLE" }); + const freshProposal = await planVerificationRecovery({ + ...input, + expiresAt, + }); + const recovered = await recoverVerification({ + ...input, + expiresAt, + approvalDigest: freshProposal.approvalDigest, + attended: true, + }); + expect(recovered).toMatchObject({ + status: "committed", + candidateCommit: started.run.candidateCommit, + attemptCount: 1, + repairCount: 0, + deadlineAt: "2020-01-01T00:00:00.000Z", + }); + await expect(resumeRun(input)).rejects.toMatchObject({ + code: "VERIFICATION_RECOVERY_UNAVAILABLE", + }); + await expect( + planVerificationRecovery({ ...input, expiresAt }), + ).rejects.toMatchObject({ code: "VERIFICATION_RECOVERY_UNAVAILABLE" }); + await writeFile(fixture.dockerPath, docker); + expect((await verifyRun(input)).evidence.passed).toBe(true); + expect((await reviewRun(input)).run.status).toBe("reviewed"); + const readDb = new DatabaseSync(dbPath, { readOnly: true }); + const row = readDb + .prepare( + "SELECT envelope_json FROM worker_invocations WHERE run_id = ? AND phase = 'review'", + ) + .get(input.runId) as { envelope_json: string }; + expect(row.envelope_json).toContain(expiresAt); + readDb.close(); + } finally { + await fixture.cleanup(); + } + }, + ); it.each([undefined, "p0_p1"] as const)( "retains P2 evidence with approved policy %s", async (policy) => { @@ -952,7 +1172,7 @@ writeFileSync(new URL("./baseline-started",import.meta.url),"started");setInterv taskPath: early.taskPath, runId: started.run.id, }), - ).rejects.toMatchObject({ code: "RUN_NOT_COMMITTED" }); + ).resolves.toMatchObject({ run: { status: "verified" } }); } finally { await early.cleanup(); } @@ -1018,6 +1238,26 @@ writeFileSync(new URL("./baseline-started",import.meta.url),"started");setInterv }), ).rejects.toMatchObject({ code: "CODEX_PROFILE_UNAVAILABLE" }); process.env.MILL_CODEX_PATH = fixture.codexPath; + const taskFile = path.join(fixture.root, fixture.taskPath); + const originalTask = await readFile(taskFile, "utf8"); + await writeFile( + taskFile, + originalTask.replace("greater than one.", "greater than two."), + ); + await expect( + reviewRun({ + root: fixture.root, + taskPath: fixture.taskPath, + runId: started.run.id, + }), + ).rejects.toMatchObject({ code: "RUN_POLICY_DRIFT" }); + expect( + (await runStatus({ root: fixture.root, runId: started.run.id })).run, + ).toMatchObject({ + status: "blocked", + blockCode: "CODEX_PROFILE_UNAVAILABLE", + }); + await writeFile(taskFile, originalTask); const reviewed = await reviewRun({ root: fixture.root, taskPath: fixture.taskPath, diff --git a/test/verification-recovery.test.ts b/test/verification-recovery.test.ts new file mode 100644 index 0000000..c56a38d --- /dev/null +++ b/test/verification-recovery.test.ts @@ -0,0 +1,132 @@ +import { describe, expect, it } from "vitest"; +import { eligibleVerificationFailure } from "../src/runtime/verification-recovery.js"; +import type { RunRecord } from "../src/runtime/state.js"; + +const run: RunRecord = { + id: "run", + repositoryId: "repo", + taskId: "task", + taskDigest: "task", + configDigest: "config", + status: "blocked", + baseCommit: "base", + candidateCommit: "candidate", + candidateTree: "tree", + deadlineAt: "2020-01-01T00:00:00.000Z", + createdAt: "2020-01-01T00:00:00.000Z", + updatedAt: "2020-01-01T00:00:00.000Z", + cancelRequested: false, + attemptCount: 1, + repairCount: 0, + blockCode: "VERIFIER_IMAGE_UNAVAILABLE", +}; +const history = [ + { + sequence: 1, + type: "candidate.committed", + data: { commit: "candidate", tree: "tree" }, + }, + { sequence: 2, type: "builder.completed", data: {} }, + { + sequence: 3, + type: "run.blocked", + data: { + from: "committed", + to: "blocked", + code: "VERIFIER_IMAGE_UNAVAILABLE", + }, + }, +]; +describe("verification recovery eligibility", () => { + it("requires the adjacent infrastructure failure, permitting only its known old rejection overwrite", () => { + expect(eligibleVerificationFailure(run, history)).toBe(3); + expect( + eligibleVerificationFailure({ ...run, blockCode: "RUN_NOT_COMMITTED" }, [ + ...history, + { + sequence: 4, + type: "run.blocker_replaced", + data: { from: "VERIFIER_IMAGE_UNAVAILABLE", to: "RUN_NOT_COMMITTED" }, + }, + ]), + ).toBe(3); + expect(() => + eligibleVerificationFailure( + { ...run, blockCode: "RUN_NOT_COMMITTED" }, + history, + ), + ).toThrow(); + }); + it.each([ + { status: "cancelled" }, + { cancelRequested: true }, + { validationJson: "{}" }, + { reviewJson: "{}" }, + { deliveryJson: "{}" }, + { remoteFeedbackJson: "{}" }, + { activeProcessId: "worker" }, + { candidateCommit: undefined }, + ] as Partial[])( + "rejects non-recoverable run state %j", + (patch) => { + expect(() => + eligibleVerificationFailure({ ...run, ...patch }, history), + ).toThrow(); + }, + ); + it.each([ + "validation.failed", + "validation.passed", + "repair.started", + "review.started", + "worker.launch_started", + "delivery.effect_intent", + "verification.recovery_applied", + ])( + "does not reach back through %s to an earlier infrastructure failure", + (type) => { + expect(() => + eligibleVerificationFailure(run, [ + ...history, + { sequence: 4, type, data: {} }, + ]), + ).toThrow(); + }, + ); + it("rejects an unrelated or malformed failure chain", () => { + expect(() => + eligibleVerificationFailure( + { ...run, candidateCommit: "different" }, + history, + ), + ).toThrow(); + expect(() => + eligibleVerificationFailure(run, [ + ...history, + { + sequence: 4, + type: "run.blocker_replaced", + data: { from: "OTHER", to: "RUN_NOT_COMMITTED" }, + }, + ]), + ).toThrow(); + expect(() => + eligibleVerificationFailure( + run, + history.map((event) => + event.sequence === 3 + ? { + ...event, + data: { + from: "committed", + to: "blocked", + code: "VALIDATION_FAILED", + }, + } + : event, + ), + ), + ).toThrow(); + expect(() => eligibleVerificationFailure(run, [])).toThrow(); + }); +}); From f5d94abe102c2527019f071ed4feac0ed9d833b6 Mon Sep 17 00:00:00 2001 From: David Ahmann <46606159+davidahmann@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:53:12 -0400 Subject: [PATCH 2/7] fix: require expired immutable authority for recovery Signed-off-by: David Ahmann <46606159+davidahmann@users.noreply.github.com> --- README.md | 3 +- architecture/ARCHITECTURE.md | 3 +- docs/verifier-recovery.md | 25 ++++++------ product/verifier-recovery.md | 6 ++- scripts/test-package.mjs | 39 +++++++++++++++++- src/runtime/lifecycle.ts | 40 +++++++++++++------ src/runtime/state.ts | 7 ++++ src/runtime/verification-recovery.ts | 4 +- test/runtime-lifecycle.test.ts | 60 ++++++++++++++++++++++++++++ 9 files changed, 156 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index d14e03b..7de6fd4 100644 --- a/README.md +++ b/README.md @@ -553,7 +553,8 @@ Cancellation records intent without discarding the receipt. Use For an eligible pre-command verifier infrastructure failure, inspect the [candidate recovery procedure](docs/verifier-recovery.md). It preserves the original run and can grant one separately approved verification/review window -for the unchanged candidate. Tests and independent review remain required. +after its original deadline expires. The unchanged candidate must have a +committed version pin. Tests and independent review remain required. Before any remote attempt, stale review scope can use `review --task product/tasks/TASK.yaml --run --refresh --base --attended`. diff --git a/architecture/ARCHITECTURE.md b/architecture/ARCHITECTURE.md index 93420d4..667ef57 100644 --- a/architecture/ARCHITECTURE.md +++ b/architecture/ARCHITECTURE.md @@ -327,7 +327,8 @@ The persisted absolute run deadline is reused for verification, review, retry, repair, and resume; ordinary checkpoints grant no fresh budget. The explicit [candidate recovery](../docs/verifier-recovery.md) boundary may grant one separately approved verification/review window after eligible infrastructure -failure. Its immutable receipt retains the original deadline, failure history, +failure and expiry of the original deadline. A committed candidate pin is +required. Its immutable receipt retains the original deadline, failure history, controller identity and exact candidate. It grants no builder or repair and requires settled worker/container ownership before application. An attempt ID plus PID, PGID, and process-start digest is diagnostic state, not signalling diff --git a/docs/verifier-recovery.md b/docs/verifier-recovery.md index c319001..f6316c9 100644 --- a/docs/verifier-recovery.md +++ b/docs/verifier-recovery.md @@ -5,10 +5,11 @@ exact digest explicitly and inspect the run's status and timeline. Mill never pulls it during verification, and restoring it does not establish passing tests. Recovery is available only for a committed candidate with an eligible -infrastructure failure. The task, configuration, frozen context, base, candidate -commit and tree must still match. Failed tests, changed files, cancellation, -uncertain workers or containers, and delivery effects require their existing -reconciliation or disposition. +infrastructure failure and a valid `mill.lock` committed in that candidate. +Unpinned candidates cannot use this recovery route. The task, configuration, +frozen context, base, candidate commit and tree must still match. Failed tests, +changed files, cancellation, uncertain workers or containers, and delivery +effects require their existing reconciliation or disposition. ## One attended recovery window @@ -27,14 +28,14 @@ millctl --json verify --task product/tasks/TASK.yaml --run millctl --json review --task product/tasks/TASK.yaml --run ``` -Inspect the exact plan before applying it. The approval binds the recorded -failure history, authority, candidate and recovery controller. Cross-version -recovery, including a repository without a version pin, requires the original -deadline to have expired, so the older pinned controller cannot start another -worker. A newer controller can service only the bound recovery operations; it -does not silently upgrade the repository's pinned tool. Preserve its exact -installed artifact and release evidence. Repin the downstream repository through -a reviewed change after the old run closes. +Inspect the exact plan before applying it. Every fresh recovery window requires +the original run deadline to have expired. This prevents an older controller +from using remaining worker authority. The approval binds the recorded failure, +immutable candidate pin, invoking checkout, authority and recovery controller. A +changed checkout pin blocks recovery. A newer controller can service only the +bound recovery operations; it does not silently upgrade the repository's tool. +Preserve its installed artifact and release evidence. Repin the downstream +repository through a reviewed change after the old run closes. The recovery grants no builder, repair, new candidate or delivery approval. P0/P1 findings stop a candidate-only allowance. Required test failures remain diff --git a/product/verifier-recovery.md b/product/verifier-recovery.md index e1a0026..d299b26 100644 --- a/product/verifier-recovery.md +++ b/product/verifier-recovery.md @@ -30,10 +30,12 @@ Acceptance: block it. - Historical overwritten blocker codes qualify only when the retained event history proves the allowed failure and excludes intervening evidence/effects. +- Recovery requires a valid version pin committed in the candidate. - A tool upgrade is explicit. Any exception to the frozen package pin binds the recovery controller version to this run and its verification/review only. - Cross-version or unpinned recovery requires the original deadline to expire, - preventing the old controller from launching a builder or repair. + Every new recovery window requires the original deadline to expire, preventing + an old controller from launching a builder or repair. Bind the immutable + candidate pin and reject invoking-checkout pin drift. - Unit/integration and installed CLI tests cover success and adversarial boundaries. Full native checks, audit and independent reviews pass before merge. Release follows the existing tag/artifact/OIDC/readback procedure. diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index 32580a5..d77d370 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -1184,6 +1184,36 @@ mill: ], recoveryFixture, ); + const recoveryTaskFile = path.join( + recoveryFixture, + "product/tasks/canary-1.yaml", + ); + await writeFile( + recoveryTaskFile, + (await readFile(recoveryTaskFile, "utf8")).replace( + "deadlineSeconds: 60", + "deadlineSeconds: 20", + ), + ); + command( + gitExecutable, + ["add", "product/tasks/canary-1.yaml"], + recoveryFixture, + ); + command( + gitExecutable, + [ + "-c", + "user.name=Mill Package Test", + "-c", + "user.email=mill-package@example.invalid", + "commit", + "--no-gpg-sign", + "-m", + "test: bound packed recovery deadline", + ], + recoveryFixture, + ); const recoveryEnvironment = { ...canaryEnvironment, MILL_STATE_HOME: path.join(state, "verification-recovery"), @@ -1239,7 +1269,14 @@ mill: ); } await rm(path.join(tools, "image-unavailable")); - const recoveryExpiry = new Date(Date.now() + 50_000).toISOString(); + // A fresh allowance must never overlap the original worker authority. + await new Promise((resolve) => + setTimeout( + resolve, + Math.max(0, Date.parse(recoveryRun.deadlineAt) - Date.now() + 50), + ), + ); + const recoveryExpiry = new Date(Date.now() + 19_000).toISOString(); const recoveryProposal = recoveryCall([ "verification-recovery", "plan", diff --git a/src/runtime/lifecycle.ts b/src/runtime/lifecycle.ts index 7a84b64..d8ddfa0 100644 --- a/src/runtime/lifecycle.ts +++ b/src/runtime/lifecycle.ts @@ -52,6 +52,7 @@ import { removeVerifiedAuthorityWorktree, resetCandidateWorktree, resolveCommit, + readCommittedFile, type GitControlSnapshot, } from "./repository.js"; import { @@ -896,13 +897,30 @@ export async function qualifyBaseline(input: { async function recoveryLock( root: string, -): Promise<{ pinnedVersion: string | null; lockDigest: string | null }> { + candidateCommit: string, +): Promise<{ pinnedVersion: string; lockDigest: string }> { const lock = await readLockStatus(root); + if (!lock.found || lock.requiredVersion === undefined) + recoveryError("Recovery requires the original committed mill.lock."); + let committed: string; + try { + committed = await readCommittedFile( + root, + candidateCommit, + "mill.lock", + 256 * 1024, + ); + } catch { + recoveryError("The immutable candidate lock cannot be read."); + } + const source = await safeReadText(root, "mill.lock", 256 * 1024); + if (source !== committed) + recoveryError( + "The invoking checkout's pin differs from the immutable candidate lock.", + ); return { - pinnedVersion: lock.requiredVersion ?? null, - lockDigest: lock.found - ? canonicalDigest(await safeReadText(root, "mill.lock", 256 * 1024)) - : null, + pinnedVersion: lock.requiredVersion, + lockDigest: canonicalDigest(committed), }; } @@ -916,7 +934,7 @@ async function verificationDeadline( await enforceExactVersion(root); return persistedRunDeadline(run); } - const lock = await recoveryLock(root); + const lock = await recoveryLock(root, receipt.candidateCommit); if ( receipt.controllerVersion !== MILL_VERSION || receipt.lockDigest !== lock.lockDigest || @@ -946,14 +964,12 @@ async function createVerificationRecoveryPlan( assertEffectAllowsNewWork(run); const failureSequence = eligibleVerificationFailure(run, store.events(runId)); const candidate = await assertRunBindings(root, run, inputs); - const lock = await recoveryLock(root); - if ( - lock.pinnedVersion !== MILL_VERSION && - Date.parse(run.deadlineAt) > Date.now() - ) + const originalDeadline = Date.parse(run.deadlineAt); + if (!Number.isFinite(originalDeadline) || originalDeadline > Date.now()) recoveryError( - "Cross-version recovery requires expiration of the original builder deadline.", + "Recovery requires expiration of the original builder deadline.", ); + const lock = await recoveryLock(root, candidate.commit); const expiry = Date.parse(expiresAt); if ( !Number.isFinite(expiry) || diff --git a/src/runtime/state.ts b/src/runtime/state.ts index d0e26c5..c73799c 100644 --- a/src/runtime/state.ts +++ b/src/runtime/state.ts @@ -1162,6 +1162,13 @@ export class StateStore { if (this.verificationRecovery(id) !== undefined) recoveryError("Recovery is single use."); assertRecoveryBinding(run, plan); + if ( + !Number.isFinite(Date.parse(run.deadlineAt)) || + Date.parse(run.deadlineAt) > Date.now() + ) + recoveryError( + "Recovery requires expiration of the original builder deadline.", + ); if ( recoveryDigest(plan) !== approvalDigest || recoveryCheckpoint(this.events(id)) !== plan.checkpointDigest || diff --git a/src/runtime/verification-recovery.ts b/src/runtime/verification-recovery.ts index 1526e0d..d418110 100644 --- a/src/runtime/verification-recovery.ts +++ b/src/runtime/verification-recovery.ts @@ -23,8 +23,8 @@ export const verificationRecoverySchema = z checkpointDigest: digest, failureSequence: z.number().int().positive(), controllerVersion: z.string(), - lockDigest: digest.nullable(), - pinnedVersion: z.string().nullable(), + lockDigest: digest, + pinnedVersion: z.string().min(1), builderAttempts: z.literal(0), repairGenerations: z.literal(0), }) diff --git a/test/runtime-lifecycle.test.ts b/test/runtime-lifecycle.test.ts index 0a99d39..9713231 100644 --- a/test/runtime-lifecycle.test.ts +++ b/test/runtime-lifecycle.test.ts @@ -39,6 +39,8 @@ import { } from "../src/runtime/repository.js"; import { acquireWriterLease, StateStore } from "../src/runtime/state.js"; import { runProcess, type ActiveProcess } from "../src/runtime/process.js"; +import { recoveryDigest } from "../src/runtime/verification-recovery.js"; +import { MILL_VERSION } from "../src/version.js"; import { runtimeFixture } from "./runtime-fixture.js"; const original = { @@ -133,13 +135,52 @@ describe("local delivery lifecycle", () => { await expect( planVerificationRecovery({ ...input, expiresAt }), ).rejects.toMatchObject({ code: "VERIFICATION_RECOVERY_UNAVAILABLE" }); + const lockPath = path.join(fixture.root, "mill.lock"); + const originalLock = await readFile(lockPath, "utf8"); + await writeFile( + lockPath, + originalLock.replace('"0.8.0"', `"${MILL_VERSION}"`), + ); + await expect( + planVerificationRecovery({ ...input, expiresAt }), + ).rejects.toMatchObject({ code: "VERIFICATION_RECOVERY_UNAVAILABLE" }); const db = new DatabaseSync(dbPath); db.prepare("UPDATE runs SET deadline_at = ? WHERE id = ?").run( "2020-01-01T00:00:00.000Z", input.runId, ); db.close(); + await expect( + planVerificationRecovery({ ...input, expiresAt }), + ).rejects.toMatchObject({ code: "VERIFICATION_RECOVERY_UNAVAILABLE" }); + await writeFile(lockPath, originalLock); const proposal = await planVerificationRecovery({ ...input, expiresAt }); + const liveDeadline = new Date(Date.now() + 60000).toISOString(); + const mutationDb = new DatabaseSync(dbPath); + mutationDb + .prepare("UPDATE runs SET deadline_at = ? WHERE id = ?") + .run(liveDeadline, input.runId); + const guardedStore = await StateStore.open( + "11111111-1111-4111-8111-111111111111", + await commonGitDirectory(fixture.root), + ); + const invalidPlan = { + ...proposal.plan, + originalDeadlineAt: liveDeadline, + }; + expect(() => + guardedStore.applyVerificationRecovery( + input.runId, + invalidPlan, + recoveryDigest(invalidPlan), + ), + ).toThrow("expiration"); + expect(guardedStore.getRun(input.runId).status).toBe("blocked"); + guardedStore.close(); + mutationDb + .prepare("UPDATE runs SET deadline_at = ? WHERE id = ?") + .run("2020-01-01T00:00:00.000Z", input.runId); + mutationDb.close(); expect(proposal.plan.pinnedVersion).toBe("0.8.0"); await recoverVerification({ ...input, @@ -147,6 +188,14 @@ describe("local delivery lifecycle", () => { approvalDigest: proposal.approvalDigest, attended: true, }); + await writeFile( + lockPath, + originalLock.replace('"0.8.0"', `"${MILL_VERSION}"`), + ); + await expect(verifyRun(input)).rejects.toMatchObject({ + code: "VERIFICATION_RECOVERY_UNAVAILABLE", + }); + await writeFile(lockPath, originalLock); expect((await verifyRun(input)).evidence.passed).toBe(true); expect((await reviewRun(input)).run.status).toBe("reviewed"); expect( @@ -162,6 +211,17 @@ describe("local delivery lifecycle", () => { const fixture = await runtimeFixture(); activate(fixture); try { + await writeFile( + path.join(fixture.root, "mill.lock"), + `schemaVersion: "1"\nmill:\n package: "@davidahmann/mill"\n version: "${MILL_VERSION}"\n`, + ); + await git(fixture.root, ["add", "mill.lock"]); + await git(fixture.root, [ + "commit", + "--no-gpg-sign", + "-m", + "test: current committed pin", + ]); const started = await startLocalRun({ root: fixture.root, taskPath: fixture.taskPath, From 69812e95ad9abbf026274c411f79ee1589e09d56 Mon Sep 17 00:00:00 2001 From: David Ahmann <46606159+davidahmann@users.noreply.github.com> Date: Mon, 21 Sep 2026 11:54:14 -0400 Subject: [PATCH 3/7] test: use explicit timer in recovery canary Signed-off-by: David Ahmann <46606159+davidahmann@users.noreply.github.com> --- scripts/test-package.mjs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index d77d370..fe3975a 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -1,3 +1,4 @@ +import { setTimeout as delay } from "node:timers/promises"; import { createHash } from "node:crypto"; import { chmod, @@ -1270,11 +1271,8 @@ mill: } await rm(path.join(tools, "image-unavailable")); // A fresh allowance must never overlap the original worker authority. - await new Promise((resolve) => - setTimeout( - resolve, - Math.max(0, Date.parse(recoveryRun.deadlineAt) - Date.now() + 50), - ), + await delay( + Math.max(0, Date.parse(recoveryRun.deadlineAt) - Date.now() + 50), ); const recoveryExpiry = new Date(Date.now() + 19_000).toISOString(); const recoveryProposal = recoveryCall([ From 903a285a54593277df2b30e07ad7cce1d251cf9f Mon Sep 17 00:00:00 2001 From: David Ahmann <46606159+davidahmann@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:04:53 -0400 Subject: [PATCH 4/7] fix(runtime): authorize recovery before writable admission Signed-off-by: David Ahmann <46606159+davidahmann@users.noreply.github.com> --- scripts/test-package.mjs | 33 ++++++ src/runtime/lifecycle.ts | 42 +++++++- test/verification-admission.test.ts | 161 ++++++++++++++++++++++++++++ 3 files changed, 235 insertions(+), 1 deletion(-) create mode 100644 test/verification-admission.test.ts diff --git a/scripts/test-package.mjs b/scripts/test-package.mjs index fe3975a..125ab4b 100644 --- a/scripts/test-package.mjs +++ b/scripts/test-package.mjs @@ -1,3 +1,4 @@ +import { existsSync } from "node:fs"; import { setTimeout as delay } from "node:timers/promises"; import { createHash } from "node:crypto"; import { @@ -1215,6 +1216,38 @@ mill: ], recoveryFixture, ); + const pinnedSource = await readFile( + path.join(recoveryFixture, "mill.lock"), + "utf8", + ); + await writeFile( + path.join(recoveryFixture, "mill.lock"), + pinnedSource.replace(MILL_VERSION, "0.8.0"), + ); + const deniedState = path.join(state, "denied-controller"); + const deniedController = spawnSync( + bin, + [ + "--json", + "--cwd", + recoveryFixture, + "verify", + "--task", + "product/tasks/canary-1.yaml", + "--run", + "11111111-1111-4111-8111-111111111111", + ], + { + cwd: recoveryFixture, + env: { ...canaryEnvironment, MILL_STATE_HOME: deniedState }, + encoding: "utf8", + timeout: 120_000, + }, + ); + if (deniedController.status === 0 || existsSync(deniedState)) { + throw new Error("unapproved controller created state before pin rejection"); + } + await writeFile(path.join(recoveryFixture, "mill.lock"), pinnedSource); const recoveryEnvironment = { ...canaryEnvironment, MILL_STATE_HOME: path.join(state, "verification-recovery"), diff --git a/src/runtime/lifecycle.ts b/src/runtime/lifecycle.ts index d8ddfa0..90103b1 100644 --- a/src/runtime/lifecycle.ts +++ b/src/runtime/lifecycle.ts @@ -182,6 +182,41 @@ async function openRunContext( return { inputs, store, commonDirectory }; } +async function openReadOnlyRunContext( + root: string, + taskPath: string, +): Promise { + const inputs = await loadRuntimeInputs(root, taskPath); + assertBuildAuthorized(inputs); + const commonDirectory = await commonGitDirectory(root); + const store = await StateStore.openReadOnly( + inputs.config.repositoryId, + commonDirectory, + ); + if (store === undefined) + recoveryError( + "Existing operational state is required; admission creates no state.", + ); + return { inputs, store, commonDirectory }; +} + +/** Resolve the narrow version exception before opening or reconciling writable state. */ +async function admitVerificationController(input: { + root: string; + taskPath: string; + runId: string; +}): Promise { + if ((await readLockStatus(input.root)).compatible) return; + const context = await openReadOnlyRunContext(input.root, input.taskPath); + try { + const run = context.store.getRun(input.runId); + await verificationDeadline(input.root, context.store, run); + await assertRunBindings(input.root, run, context.inputs); + } finally { + context.store.close(); + } +} + function storedManifest(run: RunRecord): ContextManifest { if (run.contextJson === undefined || run.contextDigest === undefined) { throw new MillError( @@ -1009,7 +1044,7 @@ export async function planVerificationRecovery(input: { runId: string; expiresAt: string; }): Promise<{ plan: VerificationRecovery; approvalDigest: string }> { - const context = await openRunContext(input.root, input.taskPath); + const context = await openReadOnlyRunContext(input.root, input.taskPath); try { const plan = await createVerificationRecoveryPlan( input.root, @@ -1037,6 +1072,9 @@ export async function recoverVerification(input: { "Candidate recovery requires attended exact-plan approval.", ExitCode.configuration, ); + const admitted = await planVerificationRecovery(input); + if (admitted.approvalDigest !== input.approvalDigest) + recoveryError("Recovery approval does not match the read-only proposal."); const context = await openRunContext(input.root, input.taskPath); let lease: Awaited> | undefined; try { @@ -1068,6 +1106,7 @@ export async function verifyRun(input: { taskPath: string; runId: string; }): Promise<{ run: PublicRunRecord; evidence: ValidationEvidence }> { + await admitVerificationController(input); const context = await openRunContext(input.root, input.taskPath); const { inputs, store } = context; let lease: Awaited> | undefined; @@ -1166,6 +1205,7 @@ export async function reviewRun(input: { review: ReturnType; usage: ProviderUsage; }> { + await admitVerificationController(input); const context = await openRunContext(input.root, input.taskPath); const { inputs, store } = context; let lease: Awaited> | undefined; diff --git a/test/verification-admission.test.ts b/test/verification-admission.test.ts new file mode 100644 index 0000000..866de21 --- /dev/null +++ b/test/verification-admission.test.ts @@ -0,0 +1,161 @@ +import { createHash, randomUUID } from "node:crypto"; +import { chmod, mkdir, readFile, readdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { DatabaseSync } from "node:sqlite"; +import { afterEach, describe, expect, it } from "vitest"; +import { + verifyRun, + reviewRun, + planVerificationRecovery, + recoverVerification, +} from "../src/runtime/lifecycle.js"; +import { commonGitDirectory } from "../src/runtime/repository.js"; +import { repositoryStateDirectory, StateStore } from "../src/runtime/state.js"; +import { runtimeFixture } from "./runtime-fixture.js"; +const original = process.env.MILL_STATE_HOME; +const originalDocker = process.env.MILL_DOCKER_PATH; +afterEach(() => { + if (originalDocker === undefined) delete process.env.MILL_DOCKER_PATH; + else process.env.MILL_DOCKER_PATH = originalDocker; + if (original === undefined) delete process.env.MILL_STATE_HOME; + else process.env.MILL_STATE_HOME = original; +}); +const operations = [ + verifyRun, + reviewRun, + (input: { root: string; taskPath: string; runId: string }) => + planVerificationRecovery({ + ...input, + expiresAt: new Date(Date.now() + 30000).toISOString(), + }), + (input: { root: string; taskPath: string; runId: string }) => + recoverVerification({ + ...input, + expiresAt: new Date(Date.now() + 30000).toISOString(), + approvalDigest: "sha256:" + "0".repeat(64), + attended: true, + }), +]; +async function fixture() { + const result = await runtimeFixture(); + process.env.MILL_STATE_HOME = result.stateHome; + await writeFile( + path.join(result.root, "mill.lock"), + 'schemaVersion: "1"\nmill:\n package: "@davidahmann/mill"\n version: "0.8.0"\n', + ); + return result; +} +describe("read-only verification controller admission", () => { + it("rejects every recovery/version exception on absent state without creating its directory", async () => { + const f = await fixture(); + try { + const directory = repositoryStateDirectory( + "11111111-1111-4111-8111-111111111111", + await commonGitDirectory(f.root), + ); + for (const operation of operations) { + await expect( + operation({ root: f.root, taskPath: f.taskPath, runId: "missing" }), + ).rejects.toThrow(); + await expect(readdir(directory)).rejects.toMatchObject({ + code: "ENOENT", + }); + } + } finally { + await f.cleanup(); + } + }); + it("does not migrate a real legacy SQLite database on rejected admission", async () => { + const f = await fixture(); + try { + const directory = repositoryStateDirectory( + "11111111-1111-4111-8111-111111111111", + await commonGitDirectory(f.root), + ); + await mkdir(directory, { recursive: true }); + const filename = path.join(directory, "state.sqlite3"); + const db = new DatabaseSync(filename); + db.exec(`CREATE TABLE metadata (key TEXT PRIMARY KEY,value TEXT NOT NULL) STRICT; +CREATE TABLE runs (id TEXT PRIMARY KEY,repository_id TEXT NOT NULL,task_id TEXT NOT NULL,task_digest TEXT NOT NULL,config_digest TEXT NOT NULL,status TEXT NOT NULL,base_commit TEXT NOT NULL,worktree_path TEXT,context_digest TEXT,context_json TEXT,control_json TEXT,candidate_commit TEXT,candidate_tree TEXT,deadline_at TEXT NOT NULL,active_pid INTEGER,cancel_requested INTEGER NOT NULL DEFAULT 0,repair_count INTEGER NOT NULL DEFAULT 0,attempt_count INTEGER NOT NULL DEFAULT 0,block_code TEXT,validation_json TEXT,review_json TEXT,created_at TEXT NOT NULL,updated_at TEXT NOT NULL) STRICT; +CREATE TABLE run_events (sequence INTEGER PRIMARY KEY AUTOINCREMENT,run_id TEXT NOT NULL REFERENCES runs(id),occurred_at TEXT NOT NULL,type TEXT NOT NULL,data_json TEXT NOT NULL) STRICT; +CREATE TABLE baseline_qualifications (approval_digest TEXT PRIMARY KEY,repository_id TEXT NOT NULL,task_digest TEXT NOT NULL,config_digest TEXT NOT NULL,base_commit TEXT NOT NULL,evidence_digest TEXT NOT NULL,created_at TEXT NOT NULL) STRICT; +INSERT INTO metadata VALUES ('schema_version','1');`); + db.close(); + const before = await readFile(filename); + const entries = await readdir(directory); + for (const operation of operations) { + await expect( + operation({ root: f.root, taskPath: f.taskPath, runId: "missing" }), + ).rejects.toMatchObject({ code: "STATE_UPGRADE_REQUIRED" }); + expect(await readFile(filename)).toEqual(before); + expect(await readdir(directory)).toEqual(entries); + } + } finally { + await f.cleanup(); + } + }); + it("does not inspect or remove a pending OCI resource before denying wrong-version admission", async () => { + const f = await fixture(); + try { + const store = await StateStore.open( + "11111111-1111-4111-8111-111111111111", + await commonGitDirectory(f.root), + ); + const directory = store.directory; + const run = store.createRun({ + repositoryId: "11111111-1111-4111-8111-111111111111", + taskId: "test", + taskDigest: "x", + configDigest: "y", + baseCommit: "a".repeat(40), + deadlineAt: new Date(Date.now() + 30000).toISOString(), + }); + store.close(); + await mkdir(path.join(directory, "oci-resources"), { recursive: true }); + const id = randomUUID(); + const journal = path.join(directory, "oci-resources", id + ".json"); + const intent = JSON.stringify({ + schemaVersion: "1", + id, + namespace: createHash("sha256") + .update(path.resolve(directory)) + .digest("hex"), + nonce: "a".repeat(64), + kind: "verifier", + name: "mill-" + id, + controllerPid: 999999999, + createdAt: new Date().toISOString(), + launchStarted: false, + }); + await writeFile(journal, intent); + const sentinel = path.join(f.stateHome, "docker-called"); + await writeFile( + f.dockerPath, + `#!${process.execPath}\nrequire("node:fs").writeFileSync(${JSON.stringify(sentinel)},"called");process.exit(1);\n`, + ); + await chmod(f.dockerPath, 0o755); + process.env.MILL_DOCKER_PATH = f.dockerPath; + const before = await readFile(path.join(directory, "state.sqlite3")); + const entries = await readdir(directory); + for (const operation of operations) { + await expect( + operation({ root: f.root, taskPath: f.taskPath, runId: run.id }), + ).rejects.toThrow(); + expect(await readFile(journal, "utf8")).toBe(intent); + expect(await readFile(path.join(directory, "state.sqlite3"))).toEqual( + before, + ); + expect( + (await readdir(directory)).filter( + (name) => !name.endsWith("-wal") && !name.endsWith("-shm"), + ), + ).toEqual(entries); + await expect(readFile(sentinel)).rejects.toMatchObject({ + code: "ENOENT", + }); + } + } finally { + await f.cleanup(); + } + }); +}); From b8341f05126e5d266adfead306c42a892ed84ddf Mon Sep 17 00:00:00 2001 From: David Ahmann <46606159+davidahmann@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:05:08 -0400 Subject: [PATCH 5/7] docs: explain read-only recovery admission Signed-off-by: David Ahmann <46606159+davidahmann@users.noreply.github.com> --- docs/verifier-recovery.md | 5 +++++ product/verifier-recovery.md | 4 ++++ 2 files changed, 9 insertions(+) diff --git a/docs/verifier-recovery.md b/docs/verifier-recovery.md index f6316c9..2c1c695 100644 --- a/docs/verifier-recovery.md +++ b/docs/verifier-recovery.md @@ -28,6 +28,11 @@ millctl --json verify --task product/tasks/TASK.yaml --run millctl --json review --task product/tasks/TASK.yaml --run ``` +Planning reads existing state without creating or upgrading it. An incompatible +controller must validate the recovery receipt before writable state access or +resource cleanup. Apply checks attendance and approval before writable access, +then rechecks the plan under the writer lease. + Inspect the exact plan before applying it. Every fresh recovery window requires the original run deadline to have expired. This prevents an older controller from using remaining worker authority. The approval binds the recorded failure, diff --git a/product/verifier-recovery.md b/product/verifier-recovery.md index d299b26..330a004 100644 --- a/product/verifier-recovery.md +++ b/product/verifier-recovery.md @@ -36,6 +36,10 @@ Acceptance: Every new recovery window requires the original deadline to expire, preventing an old controller from launching a builder or repair. Bind the immutable candidate pin and reject invoking-checkout pin drift. +- Recovery planning and incompatible-controller admission read existing state + before any writable open, migration or resource reconciliation. Apply + validates attendance and exact approval first, then rechecks under the writer + lease. - Unit/integration and installed CLI tests cover success and adversarial boundaries. Full native checks, audit and independent reviews pass before merge. Release follows the existing tag/artifact/OIDC/readback procedure. From be46b674bca14f8d46df8a32d863ea7732a25abe Mon Sep 17 00:00:00 2001 From: David Ahmann <46606159+davidahmann@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:15:27 -0400 Subject: [PATCH 6/7] test: reduce cancellation fixture process overhead Signed-off-by: David Ahmann <46606159+davidahmann@users.noreply.github.com> --- test/recipe-integration.test.ts | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/test/recipe-integration.test.ts b/test/recipe-integration.test.ts index aad24b9..def33d2 100644 --- a/test/recipe-integration.test.ts +++ b/test/recipe-integration.test.ts @@ -147,12 +147,17 @@ async function cancellingGit(directory: string): Promise { const executable = path.join(directory, "git"); await writeFile( executable, - `#!${process.execPath} -const {spawnSync}=require("node:child_process"); -const args=process.argv.slice(2); -const result=spawnSync(${JSON.stringify(gitExecutable)},args,{env:process.env,stdio:"inherit"}); -if(result.status===0&&args.includes("commit"))process.kill(process.ppid,"SIGINT"); -process.exit(result.status??1); + `#!/bin/sh +# Avoid a Node startup for every Git inspection in this cancellation fixture. +for argument do + if [ "$argument" = commit ]; then + ${"'" + gitExecutable.replaceAll("'", "'\\''") + "'"} "$@" + result=$? + if [ "$result" -eq 0 ]; then kill -INT "$PPID"; fi + exit "$result" + fi +done +exec ${"'" + gitExecutable.replaceAll("'", "'\\''") + "'"} "$@" `, { mode: 0o755 }, ); From 26e93ee15bf746be6b2c3ee9d539404282045b3e Mon Sep 17 00:00:00 2001 From: David Ahmann <46606159+davidahmann@users.noreply.github.com> Date: Mon, 21 Sep 2026 12:25:28 -0400 Subject: [PATCH 7/7] test: bound integration fixture concurrency Signed-off-by: David Ahmann <46606159+davidahmann@users.noreply.github.com> --- vitest.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/vitest.config.ts b/vitest.config.ts index a789ef7..2135b4e 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -4,6 +4,8 @@ export default defineConfig({ cacheDir: "coverage/.vite-cache", test: { clearMocks: true, + // Lifecycle fixtures spawn Git, Node and OCI fakes; cap process fan-out. + maxWorkers: 4, exclude: ["recipes/**", "node_modules/**", "dist/**", ".mill-scratch/**"], testTimeout: 10_000, coverage: {