diff --git a/CONTRIBUTING.fr.md b/CONTRIBUTING.fr.md index 2988389..44a2542 100644 --- a/CONTRIBUTING.fr.md +++ b/CONTRIBUTING.fr.md @@ -97,10 +97,19 @@ de noms de répertoires, qui est la moitié qu'une table se tromperait en premie CI.** Tous les autres gates d'ici ajoutent de la preuve ; celui-ci en retire, et cela inverse le mode de défaillance : une règle fausse est silencieuse. La CI inchangée, le pire cas d'une règle fausse est une pull request rouge, ce pour -quoi la CI existe. Ce qu'aucun test n'attrape, c'est une règle qui nomme un vrai -répertoire et prescrit la mauvaise jambe : les deux pourritures mécaniques sont -gardées dans `tools/testplan/plan_test.go`, la sémantique ne l'est pas, et c'est -écrit plutôt que dissimulé. +quoi la CI existe. + +**Chaque plan imprime sa propre limite, et ce n'est pas une politesse.** L'outil +s'est trompé cinq fois en une semaine (#588). Quatre fois du côté cher, ce qui +est survivable ; la cinquième, il a nommé quatre exécutions et 27 specs, toutes +vertes, alors que la jambe qui reproduit le défaut n'était dans aucune. Pris +pour un plafond, ce plan disait que le travail était fait. Un plan se termine +donc en nommant ce qu'aucune table ne peut savoir : **dans quelle population +vit le défaut**. Quatre pourritures mécaniques sont gardées dans +`tools/testplan/plan_test.go` : un chemin non trié, une règle qui ne correspond +à rien, une phrase `Unproven` que l'artefact qu'elle nomme contredit, et une +règle qui prescrit une jambe incapable de piloter ce qu'elle trie. L'erreur de +population n'est gardée par rien, et c'est écrit plutôt que dissimulé. La table qu'il lit, sous forme courte : diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 18c6ce7..087b639 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -90,10 +90,19 @@ names, which is the half of it a table would get wrong first. **The plan replaces the full pass locally. It never replaces the CI matrix.** Every other gate here adds proof; this one subtracts it, and that inverts the failure mode — a wrong rule is silent. With CI untouched, the worst case of a -wrong rule is a red pull request, which is exactly what CI is for. What no test -can catch is a rule that names a real directory and prescribes the wrong leg: -the two mechanical rots are guarded in `tools/testplan/plan_test.go`, the -semantic one is not, and it is written down rather than pretended away. +wrong rule is a red pull request, which is exactly what CI is for. + +**Every plan prints its own limit, and that is not a courtesy.** The tool was +wrong five times in one week (#588). Four erred on the expensive side, which is +survivable; the fifth named four runs and 27 specs, every one of them green, +while the leg that reproduces the defect was in none of them — read as a +ceiling, that plan said the work was done. So a plan now ends by naming what no +table can answer: **which population the defect lives in**. Four mechanical rots +are guarded in `tools/testplan/plan_test.go` — an un-triaged path, a rule +matching nothing, an `Unproven` sentence contradicted by the artefact it names, +and a rule prescribing a leg that cannot drive what it governs. The population +error is not guarded by anything, and it is written down rather than pretended +away. The table it reads, in short form: diff --git a/tools/falsify/specs/a-plan-that-cannot-rot.json b/tools/falsify/specs/a-plan-that-cannot-rot.json index 6b45228..dcbc95d 100644 --- a/tools/falsify/specs/a-plan-that-cannot-rot.json +++ b/tools/falsify/specs/a-plan-that-cannot-rot.json @@ -66,8 +66,9 @@ }, { "label": "a rule that runs several commands stops having to say what it leaves unproven, and the plan becomes a green describing a smaller world than its reader believes (#564)", + "note": "The sentence this used to mutate was the one #588 measured false — `mise run conformance:environment` proving a stored shape across a restart, against a suite that saves no state — so it is gone and this points at another rule with two runs.", "file": "tools/testplan/rules.go", - "find": "\t\tUnproven: \"a change to the stored shape is proved across a restart by `mise run conformance:environment`\",", + "find": "\t\tUnproven: \"octl reshapes its answers unless a call pins `-o raw`; the leg proves the wire, not the CLI's rendering\",", "replace": "\t\tUnproven: prepushIsTheWholeGate,", "test": "TestEveryRuleWithRunsSaysWhatItDoesNotProve" }, @@ -98,6 +99,55 @@ "find": "\t\t\tif changed[m.File] {", "replace": "\t\t\tif changed[m.File] || true {", "test": "TestAChangedFileEarnsTheSpecsThatMutateIt" + }, + { + "label": "the plan stops saying what it cannot know, so it reads as a ceiling again — which is how #521 was called done while the nightly leg kept failing (#588)", + "file": "tools/testplan/plan.go", + "find": "\tb.WriteString(cannotKnow)\n\treturn b.String()", + "replace": "\tif false {\n\t\tb.WriteString(cannotKnow)\n\t}\n\treturn b.String()", + "test": "TestEveryPlanSaysWhatItCannotKnow" + }, + { + "label": "an Unproven sentence may name an artefact and cite nothing, which is the state the #567 sentence lived in: it claimed a suite proved a stored shape across a restart, and nothing read the suite (#588)", + "file": "tools/testplan/claims.go", + "find": "\t\tif named := artefactsNamed(r.Unproven); len(named) > 0 && len(r.Cites) == 0 {", + "replace": "\t\tif named := artefactsNamed(r.Unproven); len(named) > 0 && len(r.Cites) == 0 && false {", + "test": "TestTheClaimCheckerFindsAClaimThatHasGoneFalse" + }, + { + "label": "a claim stops being read against the artefact it names, so a sentence resting on a token that has been deleted keeps reading as proof (#588)", + "file": "tools/testplan/claims.go", + "find": "\t\tif !strings.Contains(text, collapse(token)) {", + "replace": "\t\tif !strings.Contains(text, collapse(token)) && false {", + "test": "TestTheClaimCheckerFindsAClaimThatHasGoneFalse" + }, + { + "label": "a claim of absence stops being checked — the exact half that would have caught #567, where up.sh contains neither `snapshot` nor `--state` and a rule said it proved a restart (#588)", + "file": "tools/testplan/claims.go", + "find": "\t\tif strings.Contains(text, collapse(token)) {", + "replace": "\t\tif strings.Contains(text, collapse(token)) && false {", + "test": "TestTheClaimCheckerFindsAClaimThatHasGoneFalse" + }, + { + "label": "a real claim of this table is pointed at the wrong artefact: leg.sh does contain the word `snapshot`, so the sentence about the environment suite becomes false and must be caught against the committed table rather than against a fixture (#588)", + "file": "tools/testplan/rules.go", + "find": "\t\t\tIn: \"tools/conformance/environment/up.sh\",\n\t\t\tAbsent: []string{\"--state\", \"snapshot\"},", + "replace": "\t\t\tIn: \"tools/conformance/leg.sh\",\n\t\t\tAbsent: []string{\"--state\", \"snapshot\"},", + "test": "TestEveryUnprovenClaimHoldsAgainstTheArtefactItNames" + }, + { + "label": "the stack gate is routed to `conformance:leg -- fields` again, which is #588's second error verbatim: a real leg, which runs, and which has no machine runtime — the one population that gate refuses to work in (#566, #477)", + "file": "tools/testplan/rules.go", + "find": "\t\tRuns: []string{\"FEINT_VM=incus-ovn mise run conformance:functional\"},\n\t\tUnproven: \"three passes on one host", + "replace": "\t\tRuns: []string{\"conformance:leg -- fields\"},\n\t\tUnproven: \"three passes on one host", + "test": "TestARuleMayNotPrescribeARunThatCannotDriveWhatItGoverns" + }, + { + "label": "the runtime leg is prescribed without a runtime, so every rule naming it sends the reader to a leg leg.sh refuses at second zero (#459, #588)", + "file": "tools/testplan/plan.go", + "find": "const runtimeLeg = \"FEINT_VM=incus-ovn mise run conformance:leg -- runtime\"", + "replace": "const runtimeLeg = \"mise run conformance:leg -- runtime\"", + "test": "TestARuleMayNotPrescribeARunThatCannotDriveWhatItGoverns" } ] } diff --git a/tools/testplan/claims.go b/tools/testplan/claims.go new file mode 100644 index 0000000..de29f13 --- /dev/null +++ b/tools/testplan/claims.go @@ -0,0 +1,160 @@ +package main + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// What an `Unproven` sentence rests on, so that a sentence which has gone false +// reddens instead of being believed. +// +// THE DEFECT THIS IS. On #567 a rule said, in its own `Unproven`, that "a change +// to the stored shape is proved across a restart by `mise run +// conformance:environment`". That suite is tools/conformance/environment/up.sh, +// and it contains **zero** occurrences of `snapshot` and of `--state`: it starts +// an emulator, asserts its ready conditions, stops it, and its fixture declares +// no infrastructure at all. The sentence was false on the day it was written and +// it was greppable on that day — which is the whole argument for this file. Two +// of the five errors #588 counts are contradicted by a rule's own prose, and +// prose is the one part of this table nothing was reading. +// +// So a sentence that names an artefact of this repository must say what makes it +// true, and that "what" is read off the artefact rather than believed. +// +// WHAT THIS DOES NOT DO, because a mechanism that oversells itself is the defect +// it exists to catch: +// +// - It cannot judge a sentence that names no artefact. "a bridge is a +// different verdict for isolation alone" (#574, wrong: three declared +// capabilities turn on the mode) names nothing greppable, and no arrangement +// of this file would have caught it. It was corrected by hand. +// - A claim can be made vacuous by choosing a token every file contains. What +// stops that is not this code: it is that writing the claim at all means +// opening the artefact, which is exactly the step whose absence produced the +// false sentence above. +// - Containment is whole-file. A claim that leg.sh names a script does not +// prove the script sits in the arm the sentence means. Where that mattered, +// the token was chosen to include its context (`FEINT_FIELD_GATE=1 +// tools/conformance/score.sh`). +// - One claim discharges a sentence, even one naming two artefacts. The +// forcing rule is a prompt, not a proof. +type claim struct { + // About is the fragment of the rule's `Unproven` this holds up. It must be + // a substring of it, so that editing the sentence out from under a claim is + // itself a failure rather than a silent orphan. + About string + // In is the repository-relative artefact whose text decides the claim. + In string + // Shows are tokens that must appear in In. Whitespace is collapsed on both + // sides before matching, so gofmt realigning a struct literal does not turn + // a true claim red. + Shows []string + // Absent are tokens that must not appear in In. This is the half that + // catches the #567 sentence, and the half that will redden the day somebody + // makes it true — at which point the sentence is what needs rewriting. + Absent []string +} + +// The artefacts a sentence names, extracted syntactically. +// +// Backticks are not required. They were the first shape of this and they hand +// out a dodge: drop the backticks and the sentence names nothing this can see. +// A path is a path in prose too. +var ( + artefactPath = regexp.MustCompile(`[A-Za-z0-9_][A-Za-z0-9_./-]*\.(?:sh|go|py|json|ya?ml|toml|md)\b`) + artefactTask = regexp.MustCompile(`mise run ([a-z0-9][a-z0-9:_-]*)`) + artefactBare = regexp.MustCompile("`([a-z0-9][a-z0-9-]*:[a-z0-9][a-z0-9-]*)`") +) + +// artefactsNamed lists the files and mise tasks a sentence names. +func artefactsNamed(text string) []string { + var named []string + seen := map[string]bool{} + add := func(s string) { + if s != "" && !seen[s] { + seen[s] = true + named = append(named, s) + } + } + for _, m := range artefactPath.FindAllString(text, -1) { + add(m) + } + for _, re := range []*regexp.Regexp{artefactTask, artefactBare} { + for _, m := range re.FindAllStringSubmatch(text, -1) { + add(m[1]) + } + } + return named +} + +// collapse folds every run of whitespace to one space, so a token may be +// written the way a reader would say it rather than the way gofmt aligned it. +func collapse(s string) string { return strings.Join(strings.Fields(s), " ") } + +// checkClaims reads every claim against the artefact it names and returns what +// no longer holds. An empty result is the only acceptable one; the caller is +// TestEveryUnprovenClaimHoldsAgainstTheArtefactItNames, and +// TestTheClaimCheckerFindsAClaimThatHasGoneFalse plants each kind of defect to +// prove this can find them at all. +func checkClaims(root string, rs []rule) []string { + var problems []string + for _, r := range rs { + if r.Unproven == prepushIsTheWholeGate { + if len(r.Cites) > 0 { + problems = append(problems, fmt.Sprintf( + "rule %q leaves Unproven empty and still cites %d artefact(s): a claim with no "+ + "sentence holds nothing up", r.Path, len(r.Cites))) + } + continue + } + if named := artefactsNamed(r.Unproven); len(named) > 0 && len(r.Cites) == 0 { + problems = append(problems, fmt.Sprintf( + "rule %q says something about %s and cites nothing. An Unproven sentence that names "+ + "an artefact must say what makes it true: the one that did not (#567) claimed a "+ + "suite proved a stored shape across a restart, and that suite saves no state", + r.Path, strings.Join(named, ", "))) + } + for _, c := range r.Cites { + problems = append(problems, checkClaim(root, r, c)...) + } + } + return problems +} + +func checkClaim(root string, r rule, c claim) []string { + var problems []string + if !strings.Contains(r.Unproven, c.About) { + return []string{fmt.Sprintf( + "rule %q cites %q as holding up %q, which is not a fragment of its Unproven any more: "+ + "the sentence moved and the claim stayed", r.Path, c.In, c.About)} + } + if len(c.Shows)+len(c.Absent) == 0 { + return []string{fmt.Sprintf( + "rule %q cites %s and asks nothing of it; a citation that reads nothing is a comment", + r.Path, c.In)} + } + body, err := os.ReadFile(filepath.Join(root, c.In)) //nolint:gosec // a path from this repository's own table + if err != nil { + return []string{fmt.Sprintf( + "rule %q rests on %s, which cannot be read: %v", r.Path, c.In, err)} + } + text := collapse(string(body)) + for _, token := range c.Shows { + if !strings.Contains(text, collapse(token)) { + problems = append(problems, fmt.Sprintf( + "rule %q says %q, and %s does not contain %q — the sentence rests on something that "+ + "is no longer there", r.Path, c.About, c.In, token)) + } + } + for _, token := range c.Absent { + if strings.Contains(text, collapse(token)) { + problems = append(problems, fmt.Sprintf( + "rule %q says %q, and %s now contains %q — the artefact grew what the sentence says "+ + "it lacks, so the sentence is what needs rewriting", r.Path, c.About, c.In, token)) + } + } + return problems +} diff --git a/tools/testplan/plan.go b/tools/testplan/plan.go index 3eb5da4..4335e5e 100644 --- a/tools/testplan/plan.go +++ b/tools/testplan/plan.go @@ -251,9 +251,41 @@ func (p plan) String() string { fmt.Fprintf(&b, " · %s\n", u) } } + b.WriteString(cannotKnow) return b.String() } +// What every plan ends on, whatever it contains. +// +// This tool was wrong five times in one week (#588). Four erred on the +// expensive side, which is survivable. The fifth was #521: it named `probe`, +// `fields`, `runtime`, `environment` and 27 specs, every one of them green, +// while the defect lived in a population none of them carried — the ssh suites +// are in no leg, and `-- runtime` runs the network suites, which end on their +// own `feint clean` and sweep the leak before any closing doorstep can see it. +// Read as a ceiling, that plan said the work was done. +// +// Nothing mechanical sees that: the rule named a real directory and prescribed +// a leg that runs. So the property has to live in the output. It said "a floor, +// not a ceiling" once, in prose, in CONTRIBUTING.md, and every brief of that +// week had to repeat it by hand — which is this repository's own rule about +// comments, applied to a tool's documentation. +// +// The wording is deliberate and it is not "this plan is a floor". A clause of +// that shape is a manner of speaking and gets skimmed; what does not get +// skimmed is the name of the thing the reader is about to get wrong. So the +// first sentence names what no table can answer — which population the defect +// lives in — the second is the dated measurement rather than a caution, and +// "floor" arrives last, as the conclusion of the two. +// +// TestEveryPlanSaysWhatItCannotKnow fails without this line, in all three of a +// plan's shapes: runs, nothing-to-run, and un-triaged. +const cannotKnow = "\n" + + "What this plan cannot know is which population the defect lives in. It routes\n" + + "on the paths a diff touched: on #521 it named four runs and 27 specs, all\n" + + "green, while the leg that reproduces the defect was in none of them. So it is\n" + + "a floor — add whatever you know that a path cannot say.\n" + // resolve turns the requested base into one this clone actually has. // // `origin/main` is the right default and it is not always present: a fresh diff --git a/tools/testplan/plan_test.go b/tools/testplan/plan_test.go index 36613df..33957c6 100644 --- a/tools/testplan/plan_test.go +++ b/tools/testplan/plan_test.go @@ -501,3 +501,318 @@ func TestTheRealSpecsAreReadAndOnlyTheMatchingOnesAreEarned(t *testing.T) { t.Errorf("a file no spec names earned %v", none) } } + +// MECHANISM ONE OF #588, and the one that had to ship whatever else did. +// +// This tool was wrong five times in one week. Four erred expensively, which is +// survivable; the fifth (#521) named four runs and 27 specs, all of which went +// green, while the leg that reproduces the defect was in none of them. Read as +// a ceiling, that plan said the work was done — and the property that would +// have stopped it was written once, in prose, in CONTRIBUTING.md, where the +// reader about to act had not looked. +// +// So it is printed. In every shape a plan takes, including the two that read +// most like an all-clear: "nothing beyond prepush", and a plan whose runs are +// all cheap. The un-triaged shape carries it too, because a reader who is being +// told to add a rule is exactly a reader deciding what to run. +func TestEveryPlanSaysWhatItCannotKnow(t *testing.T) { + // Three populations, chosen because they are the three branches of + // String(): a plan with runs, a plan with none, and a plan that refuses. + for _, tc := range []struct { + what string + paths []string + }{ + {"a plan with runs", []string{"internal/core/machine/incus.go"}}, + {"a plan with nothing to run", []string{"LICENSE"}}, + {"a plan that refuses an un-triaged path", []string{"quantum/teleporter.go"}}, + } { + got := build(t.TempDir(), tc.paths).String() + for _, want := range []string{"which population the defect lives in", "#521", "floor"} { + if !strings.Contains(got, want) { + t.Errorf("%s does not say %q, so it can be read as a ceiling:\n%s", tc.what, want, got) + } + } + } + // The control, and it is what stops this passing over a String() that + // prints the notice and nothing else: the plan must still say what it does + // prescribe. + if runs := build(t.TempDir(), []string{"internal/core/machine/incus.go"}).String(); !strings.Contains(runs, "conformance:leg -- runtime") { + t.Fatalf("the plan lost its runs while gaining its closing line:\n%s", runs) + } +} + +// MECHANISM TWO OF #588: an `Unproven` sentence is checked against the artefact +// it names. +// +// The sentence that opened this: "a change to the stored shape is proved across +// a restart by `mise run conformance:environment`" (#567). That suite is +// tools/conformance/environment/up.sh, and it contains no `snapshot` and no +// `--state` — it was false the day it was written, and greppable that same day. +func TestEveryUnprovenClaimHoldsAgainstTheArtefactItNames(t *testing.T) { + problems := checkClaims(root(t), rules) + if len(problems) > 0 { + t.Errorf("%d claim(s) no longer hold:\n %s", len(problems), strings.Join(problems, "\n ")) + } + // A control against the whole mechanism being asleep: the real table must + // actually carry claims, or the test above passes over an empty world in + // the most literal way. + cited := 0 + for _, r := range rules { + cited += len(r.Cites) + } + if cited == 0 { + t.Fatal("no rule cites anything; this test would pass over a table that reads nothing") + } +} + +// The witness this repository's own skill asks for: a control whose success is +// "nothing was found" must first be shown to be able to find. Each planted +// defect below is one of the ways a claim goes false, and the last of them is +// the one #567 was. +func TestTheClaimCheckerFindsAClaimThatHasGoneFalse(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "artefact.sh"), []byte("#!/bin/sh\necho hello world\n"), 0o644); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + what string + rule rule + want string + }{ + { + what: "a sentence that names an artefact and cites nothing", + rule: rule{Path: "x/", Unproven: "this is proved by `mise run conformance:environment`"}, + want: "cites nothing", + }, + { + what: "a claim whose sentence has been edited out from under it", + rule: rule{Path: "x/", Unproven: "a rewritten sentence", + Cites: []claim{{About: "what it used to say", In: "artefact.sh", Shows: []string{"hello"}}}}, + want: "not a fragment of its Unproven", + }, + { + what: "a claim that asks nothing of the artefact it names", + rule: rule{Path: "x/", Unproven: "held by artefact.sh", + Cites: []claim{{About: "held by artefact.sh", In: "artefact.sh"}}}, + want: "asks nothing of it", + }, + { + what: "a claim resting on a token the artefact does not carry", + rule: rule{Path: "x/", Unproven: "held by artefact.sh", + Cites: []claim{{About: "held by artefact.sh", In: "artefact.sh", Shows: []string{"goodbye"}}}}, + want: "no longer there", + }, + { + what: "a claim naming an artefact that is not there at all", + rule: rule{Path: "x/", Unproven: "held by gone.sh", + Cites: []claim{{About: "held by gone.sh", In: "gone.sh", Shows: []string{"anything"}}}}, + want: "cannot be read", + }, + { + what: "a claim of absence the artefact has since contradicted — the #567 shape", + rule: rule{Path: "x/", Unproven: "artefact.sh says nothing about hello", + Cites: []claim{{About: "artefact.sh says nothing about hello", In: "artefact.sh", Absent: []string{"hello"}}}}, + want: "needs rewriting", + }, + } { + problems := checkClaims(dir, []rule{tc.rule}) + if len(problems) == 0 { + t.Errorf("%s was not reported; the checker cannot find what it searches for", tc.what) + continue + } + if !strings.Contains(strings.Join(problems, "\n"), tc.want) { + t.Errorf("%s was reported as %q, which does not name the defect", tc.what, problems) + } + } + // The accepting half, without which a checker that refuses everything would + // pass every case above and make the table unwritable. Whitespace is + // collapsed on both sides, which is what lets a claim quote a gofmt-aligned + // struct field the way a reader would say it. + held := rule{Path: "x/", Unproven: "held by artefact.sh, which says hello world and nothing about goodbye", + Cites: []claim{{ + About: "held by artefact.sh, which says hello world and nothing about goodbye", + In: "artefact.sh", + Shows: []string{"hello world"}, + Absent: []string{"goodbye"}, + }}} + if problems := checkClaims(dir, []rule{held}); len(problems) > 0 { + t.Errorf("a claim that holds was reported anyway: %q", problems) + } +} + +// MECHANISM THREE OF #588: a rule may not prescribe a run that cannot drive +// what it governs. +// +// `functional.sh` sent to `conformance:leg -- fields` (#566/#477) is the shape: +// a real leg, which runs, and which has no machine runtime — the one population +// that gate refuses to work in. TestEveryLegTheRulesNameIsALegTheScriptAccepts +// already ties leg names to leg.sh; this ties a suite's need for a runtime to +// its leg's, and it reads leg.sh for both halves rather than a list kept here. +// +// What it can see, and it is deliberately the half a machine can read without +// guessing: +// +// - a script that resolves its runtime through tools/runtime-mode.sh, which +// refuses `--vm off` by construction (asserted below, so the marker cannot +// rot silently); +// - a suite leg.sh runs only on legs leg.sh itself refuses when FEINT_VM is +// off. +// +// What it cannot see is written down in rules.go beside the four suites: the +// general property is that a prescribed run must invoke the file it was +// prescribed for, and four scripts under tools/conformance still fail it. +func TestARuleMayNotPrescribeARunThatCannotDriveWhatItGoverns(t *testing.T) { + dir := root(t) + legSh, err := os.ReadFile(filepath.Join(dir, "tools", "conformance", "leg.sh")) + if err != nil { + t.Fatal(err) + } + refused := legsRefusedWithoutARuntime(string(legSh)) + if len(refused) == 0 { + t.Fatal("no leg refuses to run without a machine runtime, which cannot be right: " + + "leg.sh refuses `runtime` rather than letting its four suites skip themselves") + } + resolver, err := os.ReadFile(filepath.Join(dir, "tools", "runtime-mode.sh")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(resolver), `if [ "$mode" = "off" ]`) { + t.Fatal("tools/runtime-mode.sh no longer refuses --vm off, so naming it no longer means " + + "a script needs a runtime; this test's whole population came from that refusal") + } + + // Half one: a run naming a leg that leg.sh refuses without a runtime must + // supply one, or the reader pastes it and is refused at second zero. + for _, r := range rules { + for _, cmd := range r.Runs { + leg, ok := legIn(cmd) + if !ok || !refused[leg] { + continue + } + if !declaresARuntime(cmd) { + t.Errorf("rule %q prescribes %q, and tools/conformance/leg.sh refuses that leg "+ + "with no FEINT_VM: the reader is sent to a run that exits 2 before it measures anything", + r.Path, cmd) + } + } + } + + // Half two: a suite that cannot be driven without a machine runtime must + // earn a run that has one. This is the #566/#477 shape, and it is checked + // against the plan rather than against the table, so a catch-all routing it + // cheaply reddens exactly as a wrong rule does. + arms := legArms(string(legSh)) + if len(arms) == 0 { + t.Fatal("no leg arm parsed out of leg.sh; this test would pass over an empty world") + } + population := 0 + for _, path := range tracked(t) { + if !strings.HasPrefix(path, "tools/conformance/") || !strings.HasSuffix(path, ".sh") { + continue + } + why := needsAMachineRuntime(dir, path, arms, refused) + if why == "" { + continue + } + population++ + got := build(dir, []string{path}) + if !anyRunDeclaresARuntime(got) { + t.Errorf("%s %s, and its plan prescribes no run that declares one:\n%s", path, why, got) + } + } + if population == 0 { + t.Fatal("no suite in tools/conformance was found to need a machine runtime; the four " + + "dataplane suites do, so this test just measured its own breakage rather than the table") + } + // The control: a suite that runs perfectly well with no runtime must not be + // dragged into the 590 s leg by this, or the cheapest way to satisfy the + // test above would be to make every rule expensive. + if why := needsAMachineRuntime(dir, "tools/conformance/scaleway/scw-cli.sh", arms, refused); why != "" { + t.Errorf("the Scaleway CLI suite was read as needing a machine runtime (%s); the whole "+ + "conformance matrix runs it with none", why) + } +} + +// legsRefusedWithoutARuntime reads leg.sh's own refusal rather than a list kept +// here: `if [ "$leg" = "runtime" ] && [ "$vm" = "off" ]`. +var legRefusal = regexp.MustCompile(`\[ "\$leg" = "([a-z0-9-]+)" \] && \[ "\$vm" = "off" \]`) + +func legsRefusedWithoutARuntime(body string) map[string]bool { + refused := map[string]bool{} + for _, m := range legRefusal.FindAllStringSubmatch(body, -1) { + refused[m[1]] = true + } + return refused +} + +// legArms maps each leg to the suites its case arm runs. Lines outside an arm +// are deliberately ignored: guard.sh and score.sh run on every leg, and +// attributing them to one would make this test claim they need a runtime. +var suitePath = regexp.MustCompile(`tools/[A-Za-z0-9_./-]+\.sh`) + +func legArms(body string) map[string][]string { + arms := map[string][]string{} + var current []string + for _, line := range strings.Split(body, "\n") { + trimmed := strings.TrimSpace(line) + switch { + case trimmed == ";;": + current = nil + continue + case strings.HasSuffix(trimmed, ")") && !strings.ContainsAny(trimmed, " \t*"): + current = strings.Split(strings.TrimSuffix(trimmed, ")"), "|") + continue + case strings.HasPrefix(trimmed, "#") || len(current) == 0: + continue + } + for _, suite := range suitePath.FindAllString(trimmed, -1) { + for _, leg := range current { + arms[leg] = append(arms[leg], suite) + } + } + } + return arms +} + +// needsAMachineRuntime answers from the artefacts, and says why when it does. +func needsAMachineRuntime(root, path string, arms map[string][]string, refused map[string]bool) string { + body, err := os.ReadFile(filepath.Join(root, path)) + if err != nil { + return "" + } + if strings.Contains(string(body), "tools/runtime-mode.sh") { + return "resolves its runtime through tools/runtime-mode.sh, which refuses --vm off" + } + carried, all := false, true + for leg, suites := range arms { + for _, suite := range suites { + if suite != path { + continue + } + carried = true + if !refused[leg] { + all = false + } + } + } + if carried && all { + return "is run by no leg but one tools/conformance/leg.sh refuses without a runtime" + } + return "" +} + +// declaresARuntime reports whether a command hands a machine runtime to what it +// starts. `FEINT_VM=off` is not one: it is the value every leg of the CI matrix +// carries, and the value the suites above refuse. +func declaresARuntime(cmd string) bool { + return strings.Contains(cmd, "FEINT_VM=") && !strings.Contains(cmd, "FEINT_VM=off") +} + +func anyRunDeclaresARuntime(p plan) bool { + for _, r := range p.Runs { + if declaresARuntime(r.Command) { + return true + } + } + return false +} diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 92aa4a6..1cfef36 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -9,7 +9,7 @@ package main // on 2026-08-27. Between those two numbers sits every wasted hour this repository // has spent re-proving parts of itself that the diff never touched. // -// Three properties make this a control rather than a suggestion, and each is held +// Five properties make this a control rather than a suggestion, and each is held // by a test in plan_test.go: // // 1. **A path nobody triaged is an error, not a cheap default.** `--check` exits @@ -28,11 +28,30 @@ package main // held by `TestAFileThatImportsTheMachineLayerEarnsTheRuntimeLeg` — which // carries its own control, a file importing nothing but fmt, so it cannot // pass over a build() that asks for the 590 s leg unconditionally. +// 4. **An `Unproven` sentence that names an artefact is read against it.** #567 +// wrote here that "a change to the stored shape is proved across a restart by +// `mise run conformance:environment`", and that suite contains no `snapshot` +// and no `--state`: false the day it was written, and greppable the same day. +// See claims.go and `TestEveryUnprovenClaimHoldsAgainstTheArtefactItNames`. +// 5. **A rule may not prescribe a run that cannot drive what it governs.** +// `functional.sh` routed to `conformance:leg -- fields` (#566/#477) is a real +// leg, which runs, and which has no machine runtime — the one population that +// gate refuses to work in. leg.sh declares both halves of that (which legs it +// refuses without a runtime, and which suites each leg runs), so +// `TestARuleMayNotPrescribeARunThatCannotDriveWhatItGoverns` reads it rather +// than a list kept here. // // `Unproven` is not decoration. The plan replaces a run that would have proved // more, and the part it drops has to be *printed* rather than quietly forgone — // otherwise this tool becomes the thing it exists to prevent: a green that // describes a smaller world than the reader believes. +// +// AND WHAT NONE OF THE FIVE SEES, because #588 counted it: a rule that names a +// real directory, prescribes a leg that runs, and is simply wrong about which +// population the defect lives in. #521 is exactly that — `-- runtime` is a real +// leg, it runs, and the network suites it carries end on their own `feint clean`, +// which sweeps the leak before any closing doorstep can see it. Nothing static +// sees that, which is why plan.go ends every plan by saying so. // A rule maps a part of the tree to the runs that judge a change to it. type rule struct { @@ -47,6 +66,11 @@ type rule struct { // Unproven is what the plan still does not establish, printed under it. // Empty means the runs are sufficient for this path, which is a claim. Unproven string + // Cites is what the Unproven sentence rests on, read off the artefacts it + // names rather than believed. A sentence naming a file or a task and citing + // nothing is a failure: see claims.go, and #567, where a rule asserted that + // a suite saving no state proved a stored shape across a restart. + Cites []claim } // prepushIsTheWholeGate is the honest answer for a path whose change cannot @@ -62,6 +86,11 @@ var rules = []rule{ Unproven: "the `fields` leg is the only one where the omission gate judges, so it is " + "required for a shape change and not merely for a new field; it does not run " + "the example stacks, the second Exoscale zone, `feint up`, or the probe", + Cites: []claim{{ + About: "the `fields` leg is the only one where the omission gate judges", + In: "tools/conformance/leg.sh", + Shows: []string{"FEINT_FIELD_GATE=1 tools/conformance/score.sh"}, + }}, }, { Path: "internal/providers/outscale/", @@ -75,6 +104,11 @@ var rules = []rule{ Runs: []string{"conformance:leg -- exo-cli", "conformance:leg -- fields"}, Unproven: "no Terraform drives this pack until upstream exoscale/terraform-provider-exoscale#573 " + "is fixed (#525), and the second zone is `mise run conformance:zones`, which no leg carries", + Cites: []claim{{ + About: "the second zone is `mise run conformance:zones`, which no leg carries", + In: "tools/conformance/leg.sh", + Absent: []string{"zones.sh"}, + }}, }, // ----------------------------------------------------------- the runtime @@ -82,20 +116,55 @@ var rules = []rule{ Path: "internal/core/machine/", Why: "the only layer that acts on the operator's own machine", Runs: []string{"FEINT_VM=incus-ovn mise run conformance:leg -- runtime"}, - Unproven: "the leg runs under OVN; a bridge is a different verdict for isolation alone, " + - "and the stacks' functional proof is `FEINT_VM=incus-ovn mise run conformance:functional`", + // This sentence used to read "a bridge is a different verdict for + // isolation alone", and that is error one of the five #588 counts: the + // bridge differed for the firewall too, which is where #574's defect + // lived. Measured on 2026-08-28 against the driver's own declaration — + // three capabilities turn on the mode, not one — and against the + // firewall's construction, which differs without any capability saying + // so. No mechanism found this: prose naming no artefact is not + // greppable, and it was corrected by reading capabilities.go. + Unproven: "the leg runs under OVN, and three declared capabilities turn on the mode — " + + "isolation, balancing and private-from-host — so a bridge is a different verdict for " + + "more than isolation. Its firewall is built differently too, with no capability " + + "saying so (docs/limits.md: the bridge-mode isolation set keeps its catch-all), and " + + "that is where #574's defect lived. The stacks' functional proof is " + + "`FEINT_VM=incus-ovn mise run conformance:functional`", + Cites: []claim{{ + About: "three declared capabilities turn on the mode — isolation, balancing and private-from-host", + In: "internal/core/machine/capabilities.go", + Shows: []string{"Isolation: d.OVN", "Balancing: d.OVN", "PrivateFromHost: !d.OVN"}, + }, { + About: "docs/limits.md: the bridge-mode isolation set keeps its catch-all", + In: "docs/limits.md", + Shows: []string{"bridge-mode isolation set keeps its catch-all"}, + }, { + About: "The stacks' functional proof is `FEINT_VM=incus-ovn mise run conformance:functional`", + In: "tools/conformance/functional.sh", + Shows: []string{"examples/stacks"}, + }}, }, { Path: "internal/core/cloudinit/", Why: "the boot payload a pack hands its machines", Runs: []string{"FEINT_VM=incus-ovn mise run conformance:leg -- runtime"}, Unproven: "a rendered document is proved by a machine that booted on it, which is `mise run conformance:ssh`", + Cites: []claim{{ + About: "a machine that booted on it, which is `mise run conformance:ssh`", + In: "tools/conformance/scaleway/ssh.sh", + Shows: []string{"sshlogin.sh"}, + }}, }, { Path: "internal/core/network/", Why: "addressing plans, subnets, firewall bindings", Runs: []string{"FEINT_VM=incus-ovn mise run conformance:leg -- runtime"}, Unproven: "isolation between two VPCs is delivered by OVN alone; under a bridge that assertion skips itself", + Cites: []claim{{ + About: "isolation between two VPCs is delivered by OVN alone", + In: "internal/core/machine/capabilities.go", + Shows: []string{"Isolation: d.OVN"}, + }}, }, // -------------------------------------------------------- the shared core @@ -103,20 +172,58 @@ var rules = []rule{ Path: "internal/core/emulator/", Why: "Env, Pack, Route, the single-port mount — all three packs ride it", Runs: []string{"conformance:leg -- probe", "conformance:leg -- fields"}, - Unproven: "a mount collision fails `NewServer` at boot, so `mise run check` already covers it; " + - "what these legs add is that the three packs still answer through it", - }, - { - Path: "internal/core/resource/", - Why: "the neutral resource every pack stores", - Runs: []string{"conformance:leg -- probe", "conformance:leg -- fields"}, - Unproven: "a change to the stored shape is proved across a restart by `mise run conformance:environment`", - }, - { - Path: "internal/core/store/", - Why: "memory store and JSON snapshot, restored as untrusted input", - Runs: []string{"conformance:leg -- probe", "conformance:environment"}, - Unproven: "a snapshot written by one version and read by another is not exercised by any leg", + Unproven: "a mount collision fails `NewServer` at boot and TestConflictingRoutesAreRejected " + + "holds it, so `mise run check` already covers it; what these legs add is that the three " + + "packs still answer through it", + Cites: []claim{{ + About: "TestConflictingRoutesAreRejected holds it, so `mise run check` already covers it", + In: "internal/core/emulator/emulator_test.go", + Shows: []string{"func TestConflictingRoutesAreRejected"}, + }}, + }, + // The two rules below carried #588's third error, and the correction is + // measured rather than argued. `conformance:environment` is + // tools/conformance/environment/up.sh: it starts an emulator, checks the + // ready conditions it declared, stops it, and asserts nothing answers + // afterwards. It contains no `snapshot` and no `--state`, and its fixture + // says of itself that it declares no infrastructure — so it stores no + // resource, saves no state, and reloads none. It was named here as the proof + // of a stored shape across a restart, and it was named in the store's own + // runs. Both are gone; what replaces them is the truth, cited. + { + Path: "internal/core/resource/", + Why: "the neutral resource every pack stores", + Runs: []string{"conformance:leg -- probe", "conformance:leg -- fields"}, + Unproven: "no suite proves the stored shape across a restart: tools/conformance/environment/up.sh " + + "neither saves nor loads state, and its fixture declares no infrastructure to store. " + + "What holds the round trip is internal/core/store's own tests (#588)", + Cites: []claim{{ + About: "tools/conformance/environment/up.sh neither saves nor loads state", + In: "tools/conformance/environment/up.sh", + Absent: []string{"snapshot", "--state"}, + }, { + About: "its fixture declares no infrastructure to store", + In: "tools/conformance/environment/fixture/feint.yaml", + Absent: []string{"iac:"}, + }}, + }, + { + Path: "internal/core/store/", + Why: "memory store and JSON snapshot, restored as untrusted input", + Runs: []string{"conformance:leg -- probe"}, + Unproven: "no run here writes a snapshot and reads it back: tools/conformance/leg.sh drives " + + "`--state` on no leg, and `conformance:environment` was named in these runs until #588 " + + "measured that it saves no state either. The round trip is held by this package's own " + + "tests, same version and across versions alike", + Cites: []claim{{ + About: "tools/conformance/leg.sh drives `--state` on no leg", + In: "tools/conformance/leg.sh", + Absent: []string{"--state"}, + }, { + About: "`conformance:environment` was named in these runs until #588 measured that it saves no state either", + In: "tools/conformance/environment/up.sh", + Absent: []string{"--state", "snapshot"}, + }}, }, // ------------------------------------------------------ what describes it @@ -145,12 +252,22 @@ var rules = []rule{ Why: "the upstream surface scan, coverage and baseline", Runs: []string{"drift:check"}, Unproven: "the scan reads a vendored SDK clone; `mise run upstream:sync` decides which one", + Cites: []claim{{ + About: "`mise run upstream:sync` decides which one", + In: "mise.toml", + Shows: []string{"{{config_root}}/.upstream", "git clone --filter=blob:none"}, + }}, }, { Path: "coverage/", Why: "the versioned artefacts the gates read", Runs: []string{"drift:check", "evidence:update"}, Unproven: "`evidence:update` drives the whole pass by construction — it is the one caller allowed to write the record", + Cites: []claim{{ + About: "it is the one caller allowed to write the record", + In: "mise.toml", + Shows: []string{"FEINT_EVIDENCE_OUT"}, + }}, }, { Path: "shapes/", @@ -163,6 +280,11 @@ var rules = []rule{ Why: "the recorded exchanges, replayed as refusals", Runs: []string{"conformance:leg -- fields"}, Unproven: "the `fields` leg is where refusals.sh runs; nothing else can raise the negative axis", + Cites: []claim{{ + About: "the `fields` leg is where refusals.sh runs", + In: "tools/conformance/leg.sh", + Shows: []string{"tools/conformance/refusals.sh"}, + }}, }, // -------------------------------- what the orphan guard named, one by one @@ -179,10 +301,16 @@ var rules = []rule{ Unproven: prepushIsTheWholeGate, }, { - Path: "internal/core/sshkey/", - Why: "the keys a machine is handed at boot", - Runs: []string{"FEINT_VM=incus-ovn mise run conformance:ssh"}, - Unproven: "that suite is not part of `mise run conformance`, which must stay runnable with no runtime", + Path: "internal/core/sshkey/", + Why: "the keys a machine is handed at boot", + Runs: []string{"FEINT_VM=incus-ovn mise run conformance:ssh"}, + Unproven: "tools/conformance/leg.sh names no ssh suite, which is what keeps every leg " + + "runnable with no runtime — and it is why that run is not part of `mise run conformance`", + Cites: []claim{{ + About: "tools/conformance/leg.sh names no ssh suite", + In: "tools/conformance/leg.sh", + Absent: []string{"ssh.sh"}, + }}, }, { Path: "internal/proxy/", @@ -285,20 +413,119 @@ var rules = []rule{ Path: "tools/conformance/scaleway/", Why: "the Scaleway suites", Runs: []string{"conformance:leg -- scw-cli"}, - Unproven: "scaleway/network.sh belongs to the runtime leg, not to this one", - }, - { - Path: "tools/conformance/outscale/", - Why: "the Outscale suites", - Runs: []string{"conformance:leg -- octl", "conformance:leg -- terraform"}, - Unproven: "outscale/network.sh and outscale/balancer.sh belong to the runtime leg", + Unproven: "tools/conformance/scaleway/network.sh belongs to the runtime leg, and has its own rule below", + Cites: []claim{{ + About: "tools/conformance/scaleway/network.sh belongs to the runtime leg", + In: "tools/conformance/leg.sh", + Shows: []string{"tools/conformance/scaleway/network.sh"}, + }}, + }, + { + Path: "tools/conformance/outscale/", + Why: "the Outscale suites", + Runs: []string{"conformance:leg -- octl", "conformance:leg -- terraform"}, + Unproven: "tools/conformance/outscale/network.sh and tools/conformance/outscale/balancer.sh " + + "belong to the runtime leg, and have their own rules below", + Cites: []claim{{ + About: "tools/conformance/outscale/network.sh and tools/conformance/outscale/balancer.sh " + + "belong to the runtime leg", + In: "tools/conformance/leg.sh", + Shows: []string{"tools/conformance/outscale/network.sh", "tools/conformance/outscale/balancer.sh"}, + }}, }, { Path: "tools/conformance/exoscale/", Why: "the Exoscale suites", Runs: []string{"conformance:leg -- exo-cli", "conformance:zones"}, - Unproven: "exoscale/network.sh belongs to the runtime leg", + Unproven: "tools/conformance/exoscale/network.sh belongs to the runtime leg, and has its own rule below", + Cites: []claim{{ + About: "tools/conformance/exoscale/network.sh belongs to the runtime leg", + In: "tools/conformance/leg.sh", + Shows: []string{"tools/conformance/exoscale/network.sh"}, + }}, + }, + + // The suites only the runtime leg runs, one rule each, because the directory + // rules above prescribe client legs that do not invoke them at all (#588). + // + // This is the same category error as #566/#477 — functional.sh sent to a leg + // with no machine runtime — and it is the one shape of it a machine can see: + // leg.sh names these four files in the arm it *itself* refuses when + // FEINT_VM is off, so the leg that runs them is the leg that needs a + // runtime, and no other leg touches them. Held by + // TestARuleMayNotPrescribeARunThatCannotDriveWhatItGoverns, which reads + // leg.sh rather than this comment. + { + Path: "tools/conformance/scaleway/network.sh", + Why: "the Scaleway dataplane suite, which only the runtime leg runs", + Runs: []string{runtimeLeg}, + Unproven: prepushIsTheWholeGate, + }, + { + Path: "tools/conformance/outscale/network.sh", + Why: "the Outscale dataplane suite, which only the runtime leg runs", + Runs: []string{runtimeLeg}, + Unproven: prepushIsTheWholeGate, }, + { + Path: "tools/conformance/exoscale/network.sh", + Why: "the Exoscale dataplane suite, which only the runtime leg runs", + Runs: []string{runtimeLeg}, + Unproven: prepushIsTheWholeGate, + }, + { + Path: "tools/conformance/outscale/balancer.sh", + Why: "the balancer dataplane suite, which only the runtime leg runs", + Runs: []string{runtimeLeg}, + Unproven: "the balancer gates on capabilities.balancing, which OVN alone declares; under a bridge it skips itself", + Cites: []claim{{ + About: "capabilities.balancing, which OVN alone declares", + In: "internal/core/machine/capabilities.go", + Shows: []string{"Balancing: d.OVN"}, + }}, + }, + + // The ssh chains, which no leg carries either — found by hand while #588 was + // measuring the sentence above, not by a mechanism. `mise run conformance` + // must stay runnable with no runtime, so these three live outside every leg + // and outside that task; the only thing that drives them is + // `conformance:ssh`, and the client-leg rules above cannot. + { + Path: "tools/conformance/scaleway/ssh.sh", + Why: "the Scaleway ssh chain: a key, a server, a real login", + Runs: []string{"FEINT_VM=incus-ovn mise run conformance:ssh"}, + Unproven: prepushIsTheWholeGate, + }, + { + Path: "tools/conformance/outscale/ssh.sh", + Why: "the Outscale ssh chain: a key, a server, a real login", + Runs: []string{"FEINT_VM=incus-ovn mise run conformance:ssh"}, + Unproven: prepushIsTheWholeGate, + }, + { + Path: "tools/conformance/exoscale/ssh.sh", + Why: "the Exoscale ssh chain: a key, a server, a real login", + Runs: []string{"FEINT_VM=incus-ovn mise run conformance:ssh"}, + Unproven: prepushIsTheWholeGate, + }, + { + Path: "tools/conformance/sshlogin.sh", + Why: "the login the three ssh chains share", + Runs: []string{"FEINT_VM=incus-ovn mise run conformance:ssh"}, + Unproven: prepushIsTheWholeGate, + }, + // WHAT IS STILL MIS-ROUTED HERE, named rather than left to be discovered. + // + // The general property is *a rule must prescribe a run that actually invokes + // the file it governs*, and this table does not enforce it. Only the half a + // machine can read today is: a suite the runtime leg alone carries, which is + // the four above. By hand, four more files under the catch-all below earn + // `conformance:leg -- fields`, a leg that runs none of them: + // crash.sh, parity.sh, stacks.sh and witness.sh, each of which has a mise + // task of its own. They are wrong on the expensive-and-useless side rather + // than the dangerous one — the reader is sent to a leg that cannot go red + // for their change — and fixing them is table work #588 deliberately did not + // take on, because the issue is about what the tool says of itself. { Path: "tools/conformance/shared/", Why: "the helpers every suite sources, waiting included", @@ -317,10 +544,16 @@ var rules = []rule{ // 2026-08-28: the plan for a change to functional.sh named five falsify // specs and two legs, and never the gate itself. { - Path: "tools/conformance/functional.sh", - Why: "the stack gate — the only gate that applies the example stacks to real machines", - Runs: []string{"FEINT_VM=incus-ovn mise run conformance:functional"}, - Unproven: "three passes on one host prove what one host holds; the CI job of runtime-proof.yml is a second population and only the night plays it", + Path: "tools/conformance/functional.sh", + Why: "the stack gate — the only gate that applies the example stacks to real machines", + Runs: []string{"FEINT_VM=incus-ovn mise run conformance:functional"}, + Unproven: "three passes on one host prove what one host holds; the CI job of " + + ".github/workflows/runtime-proof.yml is a second population and only the night plays it", + Cites: []claim{{ + About: "the CI job of .github/workflows/runtime-proof.yml is a second population and only the night plays it", + In: ".github/workflows/runtime-proof.yml", + Shows: []string{"tools/conformance/functional.sh", "cron:"}, + }}, }, { Path: "tools/conformance/functionallib.sh", @@ -332,7 +565,12 @@ var rules = []rule{ Path: "tools/conformance/", Why: "the shared harness: doorstep, score, faults, refusals, stacks, functional", Runs: []string{"conformance:leg -- fields"}, - Unproven: "score.sh judges the field gate on the `fields` leg alone; on every other leg it prints and judges nothing", + Unproven: "tools/conformance/score.sh judges the field gate on the `fields` leg alone; on every other leg it prints and judges nothing", + Cites: []claim{{ + About: "tools/conformance/score.sh judges the field gate on the `fields` leg alone", + In: "tools/conformance/leg.sh", + Shows: []string{"FEINT_FIELD_GATE=1 tools/conformance/score.sh", "FEINT_FIELD_GATE=0 tools/conformance/score.sh"}, + }}, }, // Both callers of the runtime resolver, named: `evidence:update` is twenty // minutes and the stack gate is fifteen, so the cheap one is the run and the @@ -342,12 +580,26 @@ var rules = []rule{ Why: "which runtime a task answers under, and the announcement that says so (#574)", Runs: []string{"FEINT_VM=incus-ovn mise run conformance:functional"}, Unproven: "`mise run evidence:update` is the other caller, and its twenty minutes are not in this plan; tools/evidence/mode_test.go is what holds its four outcomes offline", + Cites: []claim{{ + About: "`mise run evidence:update` is the other caller", + In: "tools/evidence/mode.sh", + Shows: []string{"runtime-mode.sh"}, + }, { + About: "tools/evidence/mode_test.go is what holds its four outcomes offline", + In: "tools/evidence/mode_test.go", + Shows: []string{"func Test"}, + }}, }, { Path: "tools/falsify/", Why: "the falsification harness and its specs", Runs: []string{"falsify:lint"}, Unproven: "`falsify:lint` is the cheap half; `mise run falsify:all` replays every mutation and belongs to the night", + Cites: []claim{{ + About: "`mise run falsify:all` replays every mutation", + In: "mise.toml", + Shows: []string{"falsify.py --all tools/falsify/specs"}, + }}, }, { Path: "tools/drift/", @@ -360,12 +612,22 @@ var rules = []rule{ Why: "the documentation gates, limits acknowledgements included", Runs: []string{"docs:check", "limits:check"}, Unproven: "`limits:check` sees a section citing an issue that has since closed, and nothing else: the one limit that went false for ten days cited no issue at all", + Cites: []claim{{ + About: "`limits:check` sees a section citing an issue that has since closed, and nothing else", + In: "mise.toml", + Shows: []string{"tools/docs/limits-acks.py check"}, + }}, }, { Path: "tools/", Why: "repository tooling", Runs: nil, Unproven: "this is the one catch-all here, and it covers scripts whose own Go tests run in `mise run check`", + Cites: []claim{{ + About: "scripts whose own Go tests run in `mise run check`", + In: "mise.toml", + Shows: []string{"[tasks.check]", "go test"}, + }}, }, // ------------------------------------------------------------ everything