From eb64446a4ad53db408904e05411665074f62e4a2 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 26 Aug 2026 04:47:09 +0000 Subject: [PATCH] fix(verify): the runner shall not report success having executed zero commands (#403) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `tools/run_verification.py` skipped an artifact with no `fields.steps[].run` and counted the skip toward exit 0. Its terminal decision was `return 0 if failed_count == 0 else 1`, so a run that matched artifacts and executed ZERO commands returned 0 — and `Verification Gate (rivet-driven)` is a REQUIRED context, so it reported green having verified nothing. Measured: `--filter '(and (= type "feature") (has-field "test-name"))'` matched 64 artifacts, ran nothing, exit 0. The sibling `if not ids: return 1` had already closed "the filter matched nothing"; "matched, and nothing ran" sat open one branch below. REQ-GUARD-GATE-EVIDENCE-003 (check_evidence_steps.py) guards the artifact PLANE via a ratchet that deliberately does NOT hard-fail the skip backlog. This is the sibling RUNNER half it named as still open: a pure `decide_exit(total, passed, failed, skipped)` reds a run that verified nothing and names the cause, while a mixed run that ran something still passes — so it does not re-introduce the "fail all 85 at once" posture the ratchet avoids. - decide_exit + `--self-test` (no rivet, no shell); the load-bearing "all skipped -> 1" case pins both the exit code and the "verified nothing" reason. Mutating the guard to never fire reds the self-test. - ci.yml: Verification-runner guardrail self-test in the rivet-validate job. - check_self_test_potency.py: `rv-zero-executed-lenient` mutant (CAUGHT). - REQ-GUARD-GATE-EVIDENCE-005 + TEST-GUARD-EVIDENCE-RUNNER (release v0.43.0), tracing to -003. Blast radius: the Verification Gate default filter (v093/v0100) selects 20 type:feature artifacts, all with runnable steps, so the default gate cannot red. Evidence-step floor unchanged (78). Co-Authored-By: Claude Opus 4.8 (1M context) --- .github/workflows/ci.yml | 15 ++++ artifacts/requirements.yaml | 60 ++++++++++++++++ artifacts/verification.yaml | 52 ++++++++++++++ tools/check_self_test_potency.py | 13 ++++ tools/run_verification.py | 114 ++++++++++++++++++++++++++++++- 5 files changed, 251 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9d6c4c05..d030eebb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -669,6 +669,21 @@ jobs: - name: Evidence-step guardrail run: tools/check_evidence_steps.py --max-stepless 78 + # REQ-GUARD-GATE-EVIDENCE-005 (#403). check_evidence_steps.py above bounds + # the artifact PLANE — no new claiming artifact may arrive step-less. This + # is the sibling half, in the RUNNER: run_verification.py counted a run + # where every matched artifact was skipped toward exit 0, so a filter that + # selected only step-less artifacts reported the required + # `Verification Gate (rivet-driven)` context green having executed nothing + # (measured: 64 matched, 0 ran, exit 0). decide_exit now reds a run that + # verified nothing; the self-test is the decision table for that (no rivet, + # no shell — it runs here, on every PR, for the same reason as its + # siblings). Deliberately not a hard fail on the skip backlog: a mixed run + # still passes, so this does not re-introduce the "fail all 85 at once" + # posture the ratchet in the guardrail above exists to avoid. + - name: Verification-runner guardrail self-test + run: tools/run_verification.py --self-test + # Required-context drift (#372). Same stdlib-only, fail-closed rules as # above, and here for the same reason: it must run on the PR that edits a # workflow, and that PR need not touch anything a path filter selects. diff --git a/artifacts/requirements.yaml b/artifacts/requirements.yaml index c84435c3..94f4daa3 100644 --- a/artifacts/requirements.yaml +++ b/artifacts/requirements.yaml @@ -5565,6 +5565,66 @@ artifacts: - type: traces-to target: REQ-GUARD-GATE-EVIDENCE-001 + - id: REQ-GUARD-GATE-EVIDENCE-005 + type: requirement + title: The verification runner shall not report success having executed zero commands + description: > + Successor to REQ-GUARD-GATE-EVIDENCE-003. -003 (and its + `check_evidence_steps.py`) bound the artifact PLANE: no new claiming + artifact may arrive with no runnable step, and the existing backlog is a + declared floor that may only fall. -003 stated explicitly that whether a + present step actually RUNS, and what the runner does when it does not, is + `run_verification.py`'s own question, left open. This is that question. + + THE INSTANCE (#403, the reported title). `tools/run_verification.py` skips + an artifact with no `fields.steps[].run` and counts the skip toward exit 0. + Its terminal decision was `return 0 if failed_count == 0 else 1`, so a run + that matched artifacts and executed ZERO commands returned 0. Measured on + the live plane: `--filter '(and (= type "feature") (has-field + "test-name"))'` matched 64 artifacts, ran nothing, exit 0 — and + `Verification Gate (rivet-driven)` is a REQUIRED context, so it reported + green having verified nothing. The sibling `if not ids: return 1` had + already closed "the filter matched nothing"; "the filter matched, and + nothing ran" sat open one branch below — the same recognise-it-once shape + -003's own instance had. + + SHALL: when the runner matches a non-empty artifact set but no verification + command executes — every match was skipped — it shall exit non-zero rather + than report success. A gate that measured nothing shall not read as a pass, + and the exit reason shall name the cause ("verified nothing"), so the + required context does not go green-adjacent with a silent explanation. + + DELIBERATELY BOUNDED, so it does not re-open what -003 closed. A run where + at least one command executed and none failed still passes even if some + claiming artifact in the set was skipped: hard-failing the whole skip + backlog is precisely the "fail all 85 at once → a gate that gets disabled" + posture -003 chose a ratchet to avoid. This closes only the case where + NOTHING at all was verified. + + ORACLE: `tools/run_verification.py --self-test` — the decision table over + `decide_exit(total, passed, failed, skipped)`, run with neither rivet nor a + shell, on every PR beside its sibling guardrail self-tests. Plus the + potency mutant `rv-zero-executed-lenient` in + `tools/check_self_test_potency.py`: replacing the zero-executed guard with a + condition that never fires reds the self-test (CAUGHT), so the load-bearing + case is shown to be load-bearing rather than merely green. + + NON-VACUITY, executed not asserted: the `all skipped (nothing ran)` case + expects exit 1 where the pre-fix runner returned 0, and the verdict pins + the "verified nothing" reason text; a real pass is asserted NOT to carry + that text. Distinct inputs, distinct outputs. + + NOT CLAIMED: that a present step is the RIGHT step or that it passes — that + remains `check_verification_filters.py`'s and the per-step cargo exit's + question, -001(a)'s honest ceiling. Nor that the skip backlog is failed: it + is not, by design; bounding it stays -003's job. + status: implemented + release: v0.43.0 + tags: [process, guardrail, ci, tooling] + links: + - type: traces-to + target: REQ-GUARD-GATE-EVIDENCE-003 + - id: REQ-GUARD-SELFTEST-POTENCY-001 type: requirement title: A guard's self-test cases shall be shown load-bearing by a declared mutation it must catch diff --git a/artifacts/verification.yaml b/artifacts/verification.yaml index 1d7172b4..7177f6ea 100644 --- a/artifacts/verification.yaml +++ b/artifacts/verification.yaml @@ -5048,6 +5048,58 @@ artifacts: - type: verifies target: REQ-GUARD-GATE-EVIDENCE-003 + - id: TEST-GUARD-EVIDENCE-RUNNER + type: feature + title: The verification runner is proven to fail when it matched artifacts but executed nothing + description: > + Verifies REQ-GUARD-GATE-EVIDENCE-005. Addresses #403's reported title — + the RUNNER half, sibling to TEST-GUARD-EVIDENCE-STEPS which verifies the + PLANE half (-003). + + THE LOAD-BEARING CASE is `decide_exit(64, 0, 0, 64) -> 1`: a run that + matched 64 artifacts and executed zero commands. Against the shipped + runner before this change the same tallies returned 0 (`return 0 if + failed_count == 0 else 1`), which is why the case is written as an executed + assertion rather than prose. The verdict pins BOTH the exit code and the + "verified nothing" reason text, because a fix that returned the right code + with a silent message would still leave the required + `Verification Gate (rivet-driven)` context reading green with no + explanation on the PR. + + IT IS SHOWN NON-VACUOUS TWO WAYS. The self-test's own table contrasts the + zero-executed case (exit 1) with a real pass (exit 0, asserted NOT to carry + the "verified nothing" text) and a mixed pass-with-skips (exit 0) — the + last of which pins the DELIBERATE bound: a run that verified something does + not fail merely because a claiming artifact in the set was skipped, so this + does not re-introduce the "fail the whole backlog" posture -003's ratchet + avoids. And the second step, `check_self_test_potency.py`, carries the + `rv-zero-executed-lenient` mutant: it replaces the zero-executed guard with + a condition that never fires and asserts the self-test then FAILS (CAUGHT), + so removing the load-bearing case cannot pass silently. + + BOTH STEPS RUN FROM A CLEAN CHECKOUT — stdlib only, no rivet, no shell, no + CI-only output — so this artifact is not the permanently-FAILED shape + REQ-GUARD-GATE-EVIDENCE-004 was about. Both were EXECUTED before being + committed. + + NOT claimed: that a present step is the right step or that it passes — that + is the per-step cargo exit's question — nor that the runner fails on a + skipped claiming artifact in an otherwise-productive run; by design it does + not. + status: implemented + release: v0.43.0 + tags: [process, guardrail, ci, tooling] + fields: + method: automated-test + steps: + - run: tools/run_verification.py --self-test + - run: tools/check_self_test_potency.py + links: + - type: satisfies + target: REQ-GUARD-GATE-EVIDENCE-005 + - type: verifies + target: REQ-GUARD-GATE-EVIDENCE-005 + - id: TEST-GUARD-CONTEXT-TIMEOUT type: feature title: The job-timeout gate fails on an unbounded required job and is blind to advisory ones diff --git a/tools/check_self_test_potency.py b/tools/check_self_test_potency.py index 9542a8b7..dacabdb9 100755 --- a/tools/check_self_test_potency.py +++ b/tools/check_self_test_potency.py @@ -162,6 +162,19 @@ class Mutant: "count as zero. Caught by `REGRESSION #385(b): axiom COUNTS` (want_msg " "'axiom=1').", ), + # ── run_verification.py ──────────────────────────────────────────────── + Mutant( + tool="run_verification.py", + id="rv-zero-executed-lenient", + old=" if passed == 0:", + new=" if passed < 0:", + note="#403: a run that matched artifacts but executed zero commands (every " + "match SKIPPED) must exit 1, not report success having verified nothing. " + "`passed < 0` is never true, so the zero-executed guard never fires and " + "an all-skipped run scores 0 again. Caught by the `all skipped (nothing " + "ran)` case and the `verified nothing` message pin; survives if either " + "is dropped.", + ), ] diff --git a/tools/run_verification.py b/tools/run_verification.py index 7cb5a401..bc41ae4a 100755 --- a/tools/run_verification.py +++ b/tools/run_verification.py @@ -14,8 +14,17 @@ --shell bash Exit code: - 0 if every matched artifact's steps passed (or were skipped) - 1 if any artifact failed + 0 if at least one command executed and none failed + 1 if any artifact failed, OR if artifacts matched the filter but zero + commands ran (every match was skipped) — a gate that verified nothing + must not report success (#403) + 2 if `rivet list` itself failed + +The `if not ids: return 1` below closed "the filter matched nothing". This +closes the sibling one line down — "the filter matched, and nothing ran": +`tools/run_verification.py --filter '(and (= type "feature") (has-field +"test-name"))'` reported passed:0 failed:0 skipped:64 EXIT=0, so the REQUIRED +`Verification Gate (rivet-driven)` context went green having executed nothing. """ from __future__ import annotations @@ -74,6 +83,90 @@ def run_one_step(cmd: str, shell: str) -> bool: return proc.returncode == 0 +def decide_exit(total: int, passed: int, failed: int, skipped: int) -> tuple[int, str]: + """Return (exit_code, reason) for a completed run over a non-empty match set. + + The caller has already handled the empty match (`if not ids: return 1`), so + ``total`` here is > 0. + + Three cases, in order: + + 1. Any artifact failed -> 1. + 2. Nothing executed — every matched artifact was skipped (``passed == 0`` and + ``failed == 0``, so ``skipped == total``) -> 1. This is the #403 defect: + the gate matched artifacts, ran zero commands, and previously returned 0. + A gate that verified nothing must not report success. Left ratchet-compatible + on purpose: a *mixed* run where some command ran and some claiming artifact + was skipped still passes — bounding the skip backlog is + ``check_evidence_steps.py``'s job, not this runner's. This closes only the + hole where *nothing at all* was verified. + 3. Otherwise -> 0. + """ + if failed > 0: + return 1, f"{failed} artifact(s) failed" + if passed == 0: + # failed == 0 here, so every one of the `total` matches was skipped. + return 1, ( + f"matched {total} artifact(s) but executed 0 verification commands " + f"({skipped} skipped); refusing to report success having verified nothing" + ) + return 0, f"{passed} passed, {skipped} skipped, 0 failed" + + +# ── self-test (the decision that turns a run's tallies into an exit code) ──── +# +# Runs BEFORE the gate it guards, and without rivet or a shell, so a refactor +# that lets `decide_exit` report success over a run that executed nothing fails +# here rather than by letting the real gate pass vacuously. The load-bearing +# case is "matched, every match SKIPPED -> 1": it is the exact #403 defect, and +# it is what makes this a detector rather than an assertion. Its assertion pins +# BOTH the exit code and the reason text ("verified nothing"), because a fix that +# returned the right code with a silent reason would still leave the required +# context reading green with no explanation on the PR. +def self_test() -> int: + passed = failed = 0 + + def report(desc: str, ok: bool, got=None, want=None) -> None: + nonlocal passed, failed + if ok: + passed += 1 + print(f" ok {desc}") + else: + failed += 1 + print(f" FAIL {desc}: got {got!r}, want {want!r}") + + print("run_verification self-test") + + # (total, passed, failed, skipped) -> expected exit code + cases = [ + ("all passed", (3, 3, 0, 0), 0), + ("passed with some skipped", (5, 2, 0, 3), 0), + ("one artifact failed", (4, 3, 1, 0), 1), + ("all skipped (nothing ran)", (64, 0, 0, 64), 1), # THE #403 case + ("single skip, nothing ran", (1, 0, 0, 1), 1), + ("a failure outranks a skip", (2, 0, 1, 1), 1), + ] + for desc, (t, p, f, s), want_code in cases: + got_code, _reason = decide_exit(t, p, f, s) + report(f"decide_exit {desc}", got_code == want_code, got_code, want_code) + + # NON-VACUITY / message pin: the zero-executed verdict must say WHY, or the + # required PR context reads green-adjacent with no reason. A code-only fix + # would pass every case above and still fail here. + code, reason = decide_exit(64, 0, 0, 64) + report("zero-executed verdict is red", code == 1, code, 1) + report("zero-executed verdict names the cause", + "verified nothing" in reason, reason, "...verified nothing") + + # A passing run must NOT carry the zero-executed message. + _, ok_reason = decide_exit(3, 3, 0, 0) + report("a real pass does not claim 'verified nothing'", + "verified nothing" not in ok_reason, ok_reason, "no 'verified nothing'") + + print(f"\n{passed} passed, {failed} failed") + return 1 if failed else 0 + + def main() -> int: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( @@ -92,8 +185,16 @@ def main() -> int: default=os.environ.get("VERIFY_SHELL", "bash"), help="shell used to execute each step (default: %(default)s)", ) + parser.add_argument( + "--self-test", + action="store_true", + help="run the decision-table self-test and exit (no rivet, no shell)", + ) args = parser.parse_args() + if args.self_test: + return self_test() + result = Result(filter=args.filter) print("== rivet verification gate ==") @@ -161,7 +262,14 @@ def main() -> int: for fid in result.failed: print(f" - {fid}") - return 0 if result.failed_count == 0 else 1 + code, reason = decide_exit( + result.total, + result.passed_count, + result.failed_count, + result.skipped_count, + ) + print(reason) + return code if __name__ == "__main__":