From f5db4b530cc484d172b4bcf0e67e2ecd8d3372d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20ROBERT?= Date: Fri, 28 Aug 2026 23:16:43 +0200 Subject: [PATCH] feat(docs): the promise on line 41 is derived from a matrix that asks the doorstep, and the quick start is four commands a reader can run (#592, #593) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on `main@3b00d23`, and verified in the files before touching them: `README.md:41` promised "Run your Terraform against Scaleway, Outscale or Exoscale" while `docs/confidence.md:48` answered "A Terraform run against Exoscale — no, and refused rather than half-served", and the Exoscale pack's VetoEngine had been refusing `iac.engine: terraform` since #525 landed on 2026-08-26. Every doc gate was green throughout, and none was wrong: they compare values an artefact also holds, and a claim about capability is not one. capabilityMatrix is provider × client × mode × support × proof × reason. The proof column is resolved, never printed: a `supported` row against clientsProvenInCI — the same workflow scan the status table reads — and a `refused` row against the pack's own VetoEngine, which is the code `up` and `down` consult. Both directions, so a veto nobody wrote down and a row nothing proves are equally refused. The promise is generated from it per locale, and unownedCapabilityClaims reads every generated block of the front pages back: a sentence naming a refused pair passes only if it carries the upstream issue that would change it. Put the old sentence back and `docs --check` exits 2, naming the pair and the fix. The Quick Start taught the 0.10 sequence and was not copyable — no directory, no main.tf, no provider block, under an `Apply complete! Resources: 5 added` unreachable from it. examples/quickstart/{scaleway,outscale} are the first example now: 34 and 43 lines, `runtime: mode: off`. examples/stacks is unchanged and keeps being the qualification stack. The apply line is counted from the configuration, and tools/conformance/quickstart.sh lifts it out of the README and requires the run to print it. The quickstart directory joins the population `feint docs --check` already judges, rather than living one directory away from it. Run: prepush green, docs:check green and red on the restored sentence, conformance:quickstart, conformance:environment, conformance:leg -- probe and -- fields on 127.0.0.1:4699, lint-shell, and the falsification specs testplan named for this diff. tools/falsify/specs/capability-matrix.json is 13 mutations and every one bites; three more, planted outside the tree, redden the quickstart gate. limits:check is red on #570 and was before this branch. Assisted-by: Claude Code (claude-opus-5) --- .github/workflows/conformance.yml | 14 + CHANGELOG.fr.md | 47 ++ CHANGELOG.md | 49 ++ README.fr.md | 78 ++- README.md | 50 +- docs/clients.md | 62 ++- examples/quickstart/outscale/feint.yaml | 42 ++ examples/quickstart/outscale/main.tf | 66 +++ examples/quickstart/scaleway/feint.yaml | 40 ++ examples/quickstart/scaleway/main.tf | 62 +++ internal/cli/capability.go | 553 +++++++++++++++++++++ internal/cli/capability_claims.go | 250 ++++++++++ internal/cli/capability_claims_test.go | 358 +++++++++++++ internal/cli/capability_render.go | 357 +++++++++++++ internal/cli/capability_test.go | 314 ++++++++++++ internal/cli/docs.go | 97 +++- internal/cli/docs_banner.go | 28 +- internal/cli/docs_clients.go | 18 +- internal/cli/docs_proved.go | 139 ++++-- internal/cli/docs_proved_test.go | 40 +- internal/cli/docs_quickstart.go | 245 +++++++-- internal/cli/docs_stacks.go | 134 +++-- internal/cli/docs_stacks_test.go | 183 +++++-- internal/cli/docs_write_order_test.go | 64 +++ mise.toml | 19 + tools/conformance/quickstart.sh | 209 ++++++++ tools/falsify/specs/capability-matrix.json | 96 ++++ tools/falsify/specs/stack-proof.json | 12 +- tools/testplan/rules.go | 25 + 29 files changed, 3425 insertions(+), 226 deletions(-) create mode 100644 examples/quickstart/outscale/feint.yaml create mode 100644 examples/quickstart/outscale/main.tf create mode 100644 examples/quickstart/scaleway/feint.yaml create mode 100644 examples/quickstart/scaleway/main.tf create mode 100644 internal/cli/capability.go create mode 100644 internal/cli/capability_claims.go create mode 100644 internal/cli/capability_claims_test.go create mode 100644 internal/cli/capability_render.go create mode 100644 internal/cli/capability_test.go create mode 100644 internal/cli/docs_write_order_test.go create mode 100755 tools/conformance/quickstart.sh create mode 100644 tools/falsify/specs/capability-matrix.json diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index b01419e3..1e4ac2c8 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -401,6 +401,20 @@ jobs: if: matrix.client == 'terraform' || matrix.client == 'opentofu' run: tools/conformance/stacks.sh http://127.0.0.1:4599 + # The README's own four commands (#593): `feint up` on + # examples/quickstart, an empty second plan, `feint down`. The example a + # reader copies first is the one whose breakage they meet first, and it + # had no gate at all — a quickstart nobody runs is a README that rots. + # + # On the terraform and opentofu legs for the reason the stacks are, and + # on an emulator and a port of its own for the reason the environment + # suite has one: `feint up` and `feint down` are what it drives, so + # pointed at the shared address its cleanup would stop the emulator this + # job's other steps are measuring. + - name: Run the quick start the way the README prints it + if: matrix.client == 'terraform' || matrix.client == 'opentofu' + run: tools/conformance/quickstart.sh 127.0.0.1:4597 + # Fault injection (#26, #356): the four real clients meeting a refusal. # # On `fields` because that is the only leg with scw, octl, exo and diff --git a/CHANGELOG.fr.md b/CHANGELOG.fr.md index c7a9a3ed..78184edb 100644 --- a/CHANGELOG.fr.md +++ b/CHANGELOG.fr.md @@ -51,8 +51,55 @@ change ni l'un ni l'autre a sa place dans `git log`. passe la déclare absente 31 % du temps, là où trois passes ramènent ce risque à 3 %, pour 295 s la passe. +- **Une matrice de capacités possède chaque phrase qui revendique un client + pour un provider, et `docs:check` relit les pages (#592).** `README.md:41` + disait « Run your Terraform against Scaleway, Outscale or Exoscale » pendant + que `docs/confidence.md` disait l'inverse et que `feint up` refusait + `iac.engine: terraform` pour ce pack depuis l'arrivée de #525 le 2026-08-26. + Tous les gates de documentation sont restés verts, et aucun n'avait tort : + ils comparent des **valeurs** qu'un artefact porte aussi, et une revendication + de capacité n'en est pas une. La matrice (`internal/cli/capability.go`) est + provider × client × mode × support × **preuve** × raison, et la colonne preuve + est ce qui en fait autre chose qu'un tableau : une ligne `supported` se résout + contre le workflow de conformance qui pilote cette paire, une ligne `refused` + contre le `VetoEngine` du pack lui-même, le code que `up` et `down` + consultent, et les deux sens sont contrôlés. La promesse du README est + générée depuis elle dans les deux langues, et chaque bloc généré des pages + d'accueil est relu : une phrase qui nomme une paire refusée ne passe que si + elle nomme aussi l'issue amont qui la changerait. Code de sortie 2, comme le + reste de la chaîne. + +- **Le démarrage rapide tient en quatre commandes copiables, sur une stack assez + courte pour se lire d'un trait (#593).** Il enseignait la séquence de 0.10, + sans répertoire, sans `main.tf` et sans bloc provider, sous un + `Apply complete! Resources: 5 added` qu'on ne pouvait pas atteindre depuis + elle. `examples/quickstart/scaleway` et `examples/quickstart/outscale` sont + le premier exemple : un provider, une adresse, une machine, 34 et 43 lignes + de Terraform, `runtime: mode: off`. `examples/stacks/` ne bouge pas et garde + son rôle : c'est la stack de qualification qui a trouvé #249 et #250, et on + lui demandait d'être aussi une première lecture. La ligne d'apply est dérivée + de la configuration, et `tools/conformance/quickstart.sh` la sort du README + pour exiger que l'exécution l'imprime. La suite joue `feint up`, un second + plan vide et `feint down` sur les deux exemples, et le répertoire quickstart + rejoint la population que `feint docs --check` juge déjà : appliqué par la CI + ou déclaré avec sa raison, et épinglant le provider qui a répondu. + +- **Les blocs générés qui portent de la prose sont rendus par langue (#591).** + Le README français ouvrait sur un démarrage rapide en anglais, parce que le + générateur injectait le même bloc dans les deux pages au motif qu'une commande + ne se traduit pas. Vrai des commandes, et elles restent partagées avec la + version, l'image et le dépôt ; les phrases autour d'elles s'écrivent deux + fois. + ### Corrigé +- **Une seule exécution de `feint docs` qui changeait deux sections d'un README + n'en gardait qu'une.** La cible était écrite depuis une copie découpée en + début d'exécution, après les fonctions qui relisent le même fichier : elle + remettait leurs sections en place et annonçait un succès, et le seul symptôme + était un `docs --check` toujours rouge après une régénération qui venait de + dire `README.md updated`. Mesuré en ajoutant le bloc de promesse. + - **Un serveur créé avec son IP publique ne garde plus une interface non filtrée à côté de sa filtrée (#548).** Créé *avec* un `ip_id`, un serveur Scaleway démarre en ne portant que cette adresse : le pilote lui donne donc diff --git a/CHANGELOG.md b/CHANGELOG.md index 890d9cfd..3650a898 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,8 +65,57 @@ what this project is judged on: **a response shape a client can observe**, and is stated there too: a stack whose `project_name` is not `default` fails on the provider's own `FindExact`. +- **A capability matrix owns every sentence that claims a client for a + provider, and `docs:check` reads the pages back (#592).** `README.md:41` said + *"Run your Terraform against Scaleway, Outscale or Exoscale"* while + `docs/confidence.md` said the opposite and `feint up` had been refusing + `iac.engine: terraform` for that pack since #525 landed on 2026-08-26. Every + doc gate stayed green throughout, and none of them was wrong: they compare + **values** an artefact also holds, and a claim about capability is not one. + The matrix (`internal/cli/capability.go`) is provider × client × mode × + support × **proof** × reason, and the proof column is what makes it more than + a table: a `supported` row is resolved against the conformance workflow that + drives that pair, a `refused` row against the pack's own `VetoEngine` — the + code `up` and `down` consult — and both directions are checked, so a veto + nobody wrote down and a row nothing proves are equally refused. The README's + promise is generated from it in both languages, and every generated block of + the front pages is read back: a sentence naming a refused pair passes only if + it also names the upstream issue that would change it. Exit code 2, like the + rest of the chain. + +- **The Quick Start is four commands a reader can copy, on a stack short enough + to read whole (#593).** It taught the 0.10 sequence — `feint start`, `eval + "$(feint env scaleway)"`, `terraform apply` — with no directory, no `main.tf` + and no provider block, under an `Apply complete! Resources: 5 added` that was + not reachable from it. `examples/quickstart/scaleway` and + `examples/quickstart/outscale` are the first example now: a provider, an + address, one machine, 34 and 43 lines of Terraform, `runtime: mode: off`. + `examples/stacks/` is unchanged and keeps its job — it is the qualification + stack that found #249 and #250, and it was being asked to be a first read as + well. The apply line is derived from the configuration and + `tools/conformance/quickstart.sh` lifts it out of the README and requires the + run to print it, so the output shown is the output produced. The suite runs + `feint up`, an empty second plan and `feint down` on both examples, on the + terraform and opentofu legs, and the quickstart directory joins the population + `feint docs --check` already judges: applied by CI or declared with a reason, + and pinning the provider that answered. + +- **The generated blocks that carry prose are rendered per locale (#591).** The + French README opened with an English Quick Start — "On your machine", "In CI, + or anywhere Docker runs" — because the generator injected one block into both + pages on the rule that a command needs no translation. True of the commands, + and they are still shared along with the version, the image and the + repository; the sentences around them are written twice. + ### Fixed +- **A single `feint docs` run that changed two sections of a README kept only + one of them.** The target was written from a copy spliced at the top of the + run, after the helpers that re-read the same file and splice into what they + find — so it put their sections back and reported success, and the only + symptom was `docs --check` still red after a regeneration that had said + `README.md updated`. Measured while adding the promise block. + - **A server created with its public IP no longer keeps an unfiltered interface beside its filtered one (#548).** Created *with* an `ip_id`, a Scaleway server boots carrying only that address, so the driver gives it a diff --git a/README.fr.md b/README.fr.md index a1f00340..cf1185a3 100644 --- a/README.fr.md +++ b/README.fr.md @@ -38,28 +38,64 @@ > [!NOTE] > L'anglais est la source. Cette page est une traduction, et elle peut avoir > pris du retard : en cas de désaccord entre les deux, [la version -> anglaise](README.md) fait foi. Les blocs générés par `feint docs` — la version -> publiée, le tableau des prérequis, les commandes d'installation, les versions -> des clients, les tableaux de couverture — restent en anglais dans les deux -> pages, parce qu'ils sont rendus depuis le code plutôt qu'écrits à la main. +> anglaise](README.md) fait foi. +> +> Les blocs générés par `feint docs` sont rendus depuis le code, jamais écrits à +> la main. Ceux qui portent de la prose sont rendus par langue : l'encadré +> ci-dessus, la promesse et le démarrage rapide. Ceux qui ne sont qu'un tableau +> de valeurs (versions publiées, prérequis, commandes d'installation, versions +> des clients, tableaux de couverture) restent en anglais dans les deux pages, +> parce qu'une commande ne se traduit pas. ## Démarrage rapide -**Testez votre Terraform sur Scaleway, Outscale ou Exoscale sans compte cloud, -sans identifiants, et sans créer la moindre ressource.** + + + +**Pointez Terraform et les CLI officielles vers votre propre machine.** +Aucun compte cloud, aucun identifiant, et rien de créé nulle part. + +Terraform et OpenTofu pilotent Scaleway et Outscale. Chaque pack a en plus +son CLI officiel, et chacun d'eux pilote cet émulateur de bout en bout : + +- **Scaleway** avec `scw`. +- **Outscale** avec `octl`. +- **Exoscale** avec `exo`. Terraform et OpenTofu reviennent le jour où une + version publiée porte le correctif de + exoscale/terraform-provider-exoscale#573, que `feint up` refuse au + portillon jusque-là. + -**On your machine** — one static binary, nothing else: +**Sur votre machine**, un binaire statique et une stack assez courte pour se lire : ```bash -feint start # detaches, waits until it answers -eval "$(feint env scaleway)" # point the official client at it -terraform apply # the real Scaleway provider +brew install stephrobert/feint/feint +git clone https://github.com/stephrobert/feint +cd feint/examples/quickstart/scaleway +feint up # vérifie la station, démarre l'émulateur, applique le Terraform +feint down # détruit ce qu'il a créé, puis arrête l'émulateur ``` -**In CI, or anywhere Docker runs** — the same emulator as a service: +Ce que ces quatre commandes impriment : + +```text +Apply complete! Resources: 2 added, 0 changed, 0 destroyed. +``` + +`feint up` lit le `feint.yaml` posé à côté du Terraform : il vérifie la station, démarre l'émulateur, exporte ce dont le client officiel a besoin, lance le moteur et attend les conditions que le fichier déclare. La stack fait un serveur et une adresse, sans runtime de machines. `examples/stacks/scaleway` est celle qui ressemble à de la production, et elle est faite pour casser l'émulateur plutôt que pour se lire en premier. + +**Vous avez déjà un projet ?** Pilotez-le à la main, c'est ce qu'un `feint.yaml` vous évite d'écrire : + +```bash +feint start # se détache, attend qu'il réponde +eval "$(feint env scaleway)" # pointe le client officiel vers lui +terraform apply # le vrai provider Scaleway +``` + +**En CI, ou partout où Docker tourne**, le même émulateur en service : ```yaml services: @@ -68,24 +104,20 @@ services: ports: ["4599:4599"] ``` -The image is control-plane only and carries no `latest` tag; [docs/install.md](docs/install.md) has the GitLab form, the compose form and the signature verification. Pull it directly with `docker run --rm -p 127.0.0.1:4599:4599 ghcr.io/stephrobert/feint:v0.11.0`. +L'image ne sert que le plan de contrôle et ne porte aucun tag `latest` ; [docs/install.md](docs/install.md) donne la forme GitLab, la forme compose et la vérification de signature. Pour la tirer directement : `docker run --rm -p 127.0.0.1:4599:4599 ghcr.io/stephrobert/feint:v0.11.0`. -**In GitHub Actions without a container** — the action from the Marketplace: +**Dans GitHub Actions, sans conteneur**, l'action du Marketplace : ```yaml - uses: stephrobert/setup-feint@v1 with: version: 0.11.0 - provider: scaleway # exports what the official client needs + provider: scaleway # exporte ce dont le client officiel a besoin ``` -It installs the released binary, **verifies its checksum before running it**, and waits until the emulator answers. +Elle installe le binaire publié, **vérifie sa somme de contrôle avant de l'exécuter**, et attend que l'émulateur réponde. -```text -Apply complete! Resources: 5 added, 0 changed, 0 destroyed. -``` - ![Un job GitHub Actions qui tire l'image, y pointe le CLI Scaleway officiel et applique du Terraform — sans compte cloud et sans secret](docs/assets/ci.gif) Des pipelines à copier tels quels, sans rien à configurer ni aucun secret à @@ -93,10 +125,10 @@ ajouter : [**examples/**](examples/) — GitHub Actions, GitLab CI, et une actio `setup-feint` qui vérifie l'empreinte du binaire avant de l'exécuter. Pas de compte. Pas d'identifiants. Rien de facturé, et rien qui continue de -tourner ailleurs que sur votre machine. Le provider qui a produit cette ligne est -le vrai, celui du registre : tout l'argument de ce dépôt est qu'il ne peut pas -voir la différence, et [ce que vous pouvez valider](docs/confidence.md) dit où -cette affirmation s'arrête. +tourner ailleurs que sur votre machine. Le provider qui a imprimé la ligne +`Apply complete!` ci-dessus est le vrai, celui du registre : tout l'argument de +ce dépôt est qu'il ne peut pas voir la différence, et [ce que vous pouvez +valider](docs/confidence.md) dit où cette affirmation s'arrête. ![Démarrage de Feint, le CLI Scaleway officiel pointé dessus, et un terraform apply exécuté contre lui](docs/assets/quickstart.gif) diff --git a/README.md b/README.md index 90b67e5d..44180183 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,45 @@ ## Quick start -**Run your Terraform against Scaleway, Outscale or Exoscale without a cloud -account, without credentials, and without creating a single resource.** + + + +**Point Terraform and the official cloud CLIs at your own machine.** +No cloud account, no credentials, and nothing created anywhere. + +Terraform and OpenTofu drive Scaleway and Outscale. Each pack also has its +own official CLI, and every one of them drives this emulator end to end: + +- **Scaleway** with `scw`. +- **Outscale** with `octl`. +- **Exoscale** with `exo`. Terraform and OpenTofu join it the day a + published release carries the fix for + exoscale/terraform-provider-exoscale#573, which `feint up` refuses at the + doorstep until one does. + -**On your machine** — one static binary, nothing else: +**On your machine** — one static binary, and a stack short enough to read whole: + +```bash +brew install stephrobert/feint/feint +git clone https://github.com/stephrobert/feint +cd feint/examples/quickstart/scaleway +feint up # checks the host, starts the emulator, applies the Terraform +feint down # destroys what it created, then stops the emulator +``` + +What those four commands print: + +```text +Apply complete! Resources: 2 added, 0 changed, 0 destroyed. +``` + +`feint up` reads the `feint.yaml` beside the Terraform: it checks the host, starts the emulator, exports what the official client needs, runs the engine and waits on the conditions the file declares. The stack is one server and one address, with no machine runtime — `examples/stacks/scaleway` is the one shaped like production, and it exists to break the emulator rather than to be read first. + +**Already have a project?** Drive it by hand, which is what a `feint.yaml` saves you from writing: ```bash feint start # detaches, waits until it answers @@ -75,10 +107,6 @@ The image is control-plane only and carries no `latest` tag; [docs/install.md](d It installs the released binary, **verifies its checksum before running it**, and waits until the emulator answers. -```text -Apply complete! Resources: 5 added, 0 changed, 0 destroyed. -``` - ![A GitHub Actions job pulling the image, pointing the official Scaleway CLI at it, and applying Terraform — no cloud account and no secret](docs/assets/ci.gif) Copy-paste pipelines for both, with nothing to configure and no secret to add: @@ -86,10 +114,10 @@ Copy-paste pipelines for both, with nothing to configure and no secret to add: action that verifies the binary before it runs it. No account. No credentials. Nothing billed, and nothing left running anywhere but -your machine. The provider that produced that line is the real one from the -registry — this repository's whole argument is that it cannot tell the -difference, and [what you can validate](docs/confidence.md) says where that -claim stops. +your machine. The provider that printed the `Apply complete!` line above is the +real one from the registry — this repository's whole argument is that it cannot +tell the difference, and [what you can validate](docs/confidence.md) says where +that claim stops. ![Starting Feint, pointing the official Scaleway CLI at it, and applying a Terraform configuration against it](docs/assets/quickstart.gif) diff --git a/docs/clients.md b/docs/clients.md index 2e938d0e..e9565c25 100644 --- a/docs/clients.md +++ b/docs/clients.md @@ -32,9 +32,10 @@ same emulator you did, and its third column is doing real work: repository, and this page will not pretend otherwise. - **not pinned** — no constraint at all. Whatever the registry served that day. -The last column is derived from the conformance workflow and from -`tools/conformance/stacks.sh`, so a fixture nothing applies cannot pass for a -proof by sitting in the directory. +The last column is derived from the conformance workflow and from the suites +that apply the examples — `tools/conformance/stacks.sh` for the qualification +stacks, `tools/conformance/quickstart.sh` for the quick start ones — so a +fixture nothing applies cannot pass for a proof by sitting in the directory. Two things this table said the day it landed, which nothing had said before, and which are now checked rather than read. A stack CI applies must carry a provider @@ -46,6 +47,10 @@ in `stacksRunByHand` (`internal/cli/docs_stacks.go`); `feint docs --check` exits stack that no longer exists or that CI has started applying. The list below the table is that declaration, printed from the same source the refusal reads. +Both families are in that population, and deliberately: `examples/quickstart` +holds the example the most people copy first, and an example nobody runs is a +README that rots (#593). + ## The versions @@ -76,6 +81,8 @@ Each row is one `required_providers` entry, read where it is written. | `examples/stacks/outscale` | `outscale/outscale` | `~> 1.7` | constraint: resolved fresh on each run, so the version that answered is not knowable here | yes | | `examples/stacks/outscale/modules/net` | `outscale/outscale` | `~> 1.7` | constraint: resolved fresh on each run, so the version that answered is not knowable here | yes | | `examples/stacks/scaleway` | `scaleway/scaleway` | `2.81.0` | exact: the version that answered | yes | +| `examples/quickstart/outscale` | `outscale/outscale` | `~> 1.7` | constraint: resolved fresh on each run, so the version that answered is not knowable here | yes | +| `examples/quickstart/scaleway` | `scaleway/scaleway` | `2.81.0` | exact: the version that answered | yes | Read the third column narrowly, because it is the one a consumer pins against. An **exact** constraint names the version that answered. A **constraint** is @@ -99,6 +106,55 @@ nobody wired up: second plan, clean destroy, 2026-08-24. +## Which client may drive which pack + +The table above says in which *version* a client was proved. This one says +whether it may drive a pack at all, and it exists because of a sentence that was +false for two days on the front line of this repository (#592). + +`README.md:41` read *"Run your Terraform against Scaleway, Outscale or +Exoscale"*. `docs/confidence.md` read *"A Terraform run against Exoscale — no, +and refused rather than half-served"*. And `feint up` had been refusing +`iac.engine: terraform` for that pack since #525 landed on 2026-08-26, because +the published provider splits a single apply between this emulator and a paying +account. Every doc gate was green throughout, and none of them was wrong: they +check **values** — a coverage count, a route total, an image tag, a client +version — and each of those is something an artefact also holds. *"Run your +Terraform against three clouds"* is a **claim about capability**, and nothing +owned it. + +So the matrix below owns them, and every sentence in this repository that claims +a client for a provider is generated from it. Its `proof` column is what makes +it more than a table: a row names the instrument that establishes it, and a row +whose instrument does not carry it fails `feint docs --check` at exit code 2 — +in both directions, because a check that only refuses what is written down stops +measuring the day something stops being written down. + + + + +| Provider | Client | Mode | Support | Proof | Reason | +|---|---|---|---|---|---| +| Exoscale | `exo` | control plane | **supported** | `conformance workflow` | — | +| Exoscale | OpenTofu | control plane | **refused** | `up.go VetoEngine` | OpenTofu resolves the same published provider from the same registry namespace, so it splits the same way and waits on the same release of exoscale/terraform-provider-exoscale#573 | +| Exoscale | Terraform | control plane | **refused** | `up.go VetoEngine` | the published provider builds two clients and only one honours `EXOSCALE_API_ENDPOINT`, so an apply or a destroy splits between this emulator and a paying account (#525 counted five signed requests leaving for `api-ch-*.exoscale.com`). Upstream exoscale/terraform-provider-exoscale#573 is closed and its fix is merged into `master`; no published release carries it, the last tag being v0.70.0 of 17 July 2026 | +| Outscale | `octl` | control plane | **supported** | `conformance workflow` | — | +| Outscale | OpenTofu | control plane | **supported** | `conformance workflow` | — | +| Outscale | Terraform | control plane | **supported** | `conformance workflow` | — | +| Scaleway | OpenTofu | control plane | **supported** | `conformance workflow` | — | +| Scaleway | `scw` | control plane | **supported** | `conformance workflow` | — | +| Scaleway | Terraform | control plane | **supported** | `conformance workflow` | — | + +Every sentence in this repository that claims a client for a provider is +generated from this table, and every row is resolved against the thing its +proof column names: `conformance workflow` means +`.github/workflows/conformance.yml` runs a suite driving that client against +that pack on every pull request, and `up.go VetoEngine` means the pack itself +refuses the engine before `feint up` starts a process. A row whose proof does +not carry it fails `feint docs --check` at exit code 2, and so does a page +that claims a pair this table does not. + + ## Where else this appears The same block is spliced into the body of every GitHub Release, so the page and diff --git a/examples/quickstart/outscale/feint.yaml b/examples/quickstart/outscale/feint.yaml new file mode 100644 index 00000000..031c587f --- /dev/null +++ b/examples/quickstart/outscale/feint.yaml @@ -0,0 +1,42 @@ +# The whole environment this example needs, declared once. +# +# `feint up` reads this file and does all of it: it checks the host, starts the +# emulator, exports what the Outscale clients need, runs Terraform here, and +# waits for the conditions below against the emulator's own API. `feint down` +# takes it back down and stops the emulator. +# +# docs/environment.md is the field reference and says why each line is here. +version: 1 + +cloud: + provider: outscale + +emulator: + addr: 127.0.0.1:4599 + +# Nothing here boots a machine, and that is the point of a first example: thirty +# seconds means a control plane, an address, and a Vm the API describes. Real +# machines are the ten-minute depth, and `examples/stacks/outscale` is the stack +# written for it. +runtime: + mode: off + +iac: + engine: terraform + directory: . + vars: + # main.tf declares `endpoint` with a default of 127.0.0.1:4599. Left to that + # default against an emulator on another port, Terraform blocks to its own + # ceiling and the message blames the provider. Written once, here. + endpoint: ${feint.endpoint} + +# Asserted against the emulator, never against Terraform's state file: the Vm +# this example creates, and the public address linked to it. +# +# The Outscale API answers POST on /api/v1/, so the emulator's own +# health is what an http condition can read here; the resource counts do the +# rest, and they are read from the store rather than from the engine. +ready: + - http:/_feint/health + - resource:vm:1 + - resource:publicip:1 diff --git a/examples/quickstart/outscale/main.tf b/examples/quickstart/outscale/main.tf new file mode 100644 index 00000000..eb489965 --- /dev/null +++ b/examples/quickstart/outscale/main.tf @@ -0,0 +1,66 @@ +# The first thirty seconds: one Vm and one public address, on Outscale. +# +# The same shape as examples/quickstart/scaleway in Outscale's own vocabulary: a +# Vm rather than a server, an OMI rather than an image, and a public IP that is +# created and then *linked* rather than attached at creation — which is the one +# thing about this API a Scaleway reader gets wrong first. +# +# `examples/stacks/outscale` is the other thing (#593): two peered Nets, a NAT +# service, route tables, a balancer, machines spread over two subregions. It is +# there to try to break the emulator; this is here to be read. + +terraform { + required_version = ">= 1.7.0" + required_providers { + outscale = { + source = "outscale/outscale" + # The floor is 1.7, the generation that reads its endpoint path from the + # value (OSC_ENDPOINT_API carries /api/v1) where 1.1.x appends the path + # itself. A resolution below it would silently change how this example + # must be pointed at the emulator. + version = "~> 1.7" + } + } +} + +# Overridden by `feint up`, which passes the address it started the emulator on +# (iac.vars in feint.yaml). +variable "endpoint" { + type = string + default = "http://127.0.0.1:4599" +} + +# Their documentation gives the endpoint as +# `https://api..outscale.com/api/v1`, so the version segment belongs to +# the value rather than being appended by the provider. Getting it wrong is not +# a warning: the emulator answers 404 and names the missing prefix. +provider "outscale" { + access_key_id = "AAAAAAAAAAAAAAAAAAAA" + secret_key_id = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB" + + api { + endpoint = "${var.endpoint}/api/v1" + region = "eu-west-2" + } +} + +resource "outscale_vm" "quickstart" { + # A catalogue OMI: ami-00000003 is one of the identifiers the emulator can + # really boot. An unknown one applies fine with no machine runtime and is + # refused the moment somebody turns one on, which is a first example teaching + # a habit that breaks later. + image_id = "ami-00000003" + vm_type = "tinav6.c1r1p2" +} + +resource "outscale_public_ip" "quickstart" {} + +resource "outscale_public_ip_link" "quickstart" { + vm_id = outscale_vm.quickstart.vm_id + public_ip = outscale_public_ip.quickstart.public_ip +} + +output "address" { + description = "The public address the emulator published for the Vm." + value = outscale_public_ip.quickstart.public_ip +} diff --git a/examples/quickstart/scaleway/feint.yaml b/examples/quickstart/scaleway/feint.yaml new file mode 100644 index 00000000..9e3db3c8 --- /dev/null +++ b/examples/quickstart/scaleway/feint.yaml @@ -0,0 +1,40 @@ +# The whole environment this example needs, declared once. +# +# `feint up` reads this file and does all of it: it checks the host, starts the +# emulator, exports what the Scaleway clients need, runs Terraform here, and +# waits for the conditions below against the emulator's own API. `feint down` +# takes it back down and stops the emulator. +# +# docs/environment.md is the field reference and says why each line is here. +version: 1 + +cloud: + provider: scaleway + +emulator: + addr: 127.0.0.1:4599 + +# Nothing here boots a machine, and that is the point of a first example: thirty +# seconds means a control plane, an address, and a server the API describes. +# Real machines are the ten-minute depth — set this to incus, incus-vm or +# incus-ovn and `feint up` refuses if the host cannot deliver it, rather than +# downgrading in silence. `examples/stacks/scaleway` is the stack written for +# that. +runtime: + mode: off + +iac: + engine: terraform + directory: . + vars: + # main.tf declares `endpoint` with a default of 127.0.0.1:4599. Left to that + # default against an emulator on another port, Terraform blocks to its own + # ceiling and the message blames the provider. Written once, here. + endpoint: ${feint.endpoint} + +# Asserted against the emulator, never against Terraform's state file: the +# instance API answers, and it holds the one server this example creates. +ready: + - http:/instance/v1/zones/fr-par-1/servers + - resource:instance/server:1 + - resource:instance/ip:1 diff --git a/examples/quickstart/scaleway/main.tf b/examples/quickstart/scaleway/main.tf new file mode 100644 index 00000000..23ca4ca4 --- /dev/null +++ b/examples/quickstart/scaleway/main.tf @@ -0,0 +1,62 @@ +# The first thirty seconds: one server, one public address, on Scaleway. +# +# This is a *pedagogical* example, not a qualification stack. Its whole job is +# to be read whole before it is run, so it stops at what the Quick Start +# promises: the real Scaleway provider applies, the emulator answers, a second +# plan is empty and `feint down` destroys it. +# +# `examples/stacks/scaleway` is the other thing, and the split is deliberate +# (#593): 625 lines of two VPCs, ACLs, a bastion, a balancer, snapshots and a +# golden image, written to try to break the emulator. It found #249 and #250, +# which every other gate was blind to. It is a poor first read, and it was +# being asked to be both at once. + +terraform { + required_version = ">= 1.7.0" + required_providers { + scaleway = { + source = "scaleway/scaleway" + # Exact, and for the reason docs/clients.md gives at length: a floating + # constraint is resolved by `terraform init -upgrade` on every run, so an + # apply proves the emulator answered whatever was newest that morning and + # nothing anybody can replay. `feint docs --check` refuses an applied + # example that pins nothing. + version = "2.81.0" + } + } +} + +# Overridden by `feint up`, which passes the address it started the emulator on +# (iac.vars in feint.yaml). The default is what a hand-run `terraform apply` +# meets after `feint start`. +variable "endpoint" { + type = string + default = "http://127.0.0.1:4599" +} + +# The credentials are deliberately fake and deliberately public: the emulator +# accepts any, and `feint env scaleway` exports these same values. +provider "scaleway" { + api_url = var.endpoint + access_key = "SCWXXXXXXXXXXXXXXXXX" + secret_key = "11111111-1111-1111-1111-111111111111" + project_id = "11111111-1111-1111-1111-111111111111" + organization_id = "11111111-1111-1111-1111-111111111111" + region = "fr-par" + zone = "fr-par-1" +} + +resource "scaleway_instance_ip" "web" {} + +resource "scaleway_instance_server" "web" { + name = "quickstart-web" + type = "DEV1-S" + image = "ubuntu_jammy" + ip_id = scaleway_instance_ip.web.id + tags = ["quickstart"] +} + +output "address" { + description = "The public address the emulator published for the server." + value = scaleway_instance_ip.web.address +} diff --git a/internal/cli/capability.go b/internal/cli/capability.go new file mode 100644 index 00000000..0b61b46d --- /dev/null +++ b/internal/cli/capability.go @@ -0,0 +1,553 @@ +package cli + +import ( + "fmt" + "os" + "regexp" + "slices" + "sort" + "strings" + + "github.com/stephrobert/feint/internal/environment" +) + +// Which client drives which pack, what says so, and the sentences derived from +// it. +// +// #592 is the defect this file owns. `README.md:41` said *"Run your Terraform +// against Scaleway, Outscale or Exoscale"* while `docs/confidence.md:48` said +// *"A Terraform run against Exoscale — no, and refused rather than +// half-served"*, and `feint up` had been refusing `iac.engine: terraform` for +// that pack since #525 landed on 2026-08-26. Two days of a front line promising +// the one client the doorstep turns away, through every green `docs:check` in +// between. +// +// None of the existing gates could see it, and they are not weak: they check +// **values**. A coverage count, a route total, an image tag, a client version, +// a `feint.yaml` field — each is a number or a name an artefact also holds, so +// each can be compared. *"Run your Terraform against Scaleway, Outscale or +// Exoscale"* is a **claim about capability**, and nothing owned it. A sentence +// nobody owns cannot be contradicted. +// +// So the matrix below owns them. Three properties make it more than a table, +// and the third is the one that matters: +// +// 1. **Every row names the instrument that establishes it**, the way +// `Route.Operation` names an upstream operation. `proof` is not prose: it +// is a value capabilityProblems resolves, and a row whose instrument does +// not carry it fails its own check. +// 2. **The instruments already exist, and are asked rather than restated.** A +// `supported` row is confirmed by clientsProvenInCI — the same scan of +// .github/workflows/conformance.yml the status table and the client matrix +// read. A `refused` row is confirmed by asking the pack's own VetoEngine, +// which is the code `up` and `down` consult before a process starts. A +// matrix that disagreed with up.go would be #592 one storey lower. +// 3. **The check runs in both directions.** A row nothing establishes is +// refused, and a pair an instrument establishes with no row is refused too. +// One direction alone stops measuring the day its subject moves — the +// lesson docs_stacks.go paid for. +// +// And the sentences are derived from it: renderPromise writes the README's +// promise in both locales, so the claim on line 41 is no longer a thing anybody +// types. What guards the rest is unownedCapabilityClaims, which reads every +// generated block of the front pages and requires a matrix row behind every +// client/provider pair it finds. Put the old sentence back inside a generated +// block and `docs --check` exits 2 — +// TestTheOldFalsePromiseIsCaughtByTheClaimReader is that mutation, kept as a +// test so the reader cannot quietly stop finding it. + +const ( + capabilityStartMarker = "" + capabilityEndMarker = "" + + promiseStartMarker = "" + promiseEndMarker = "" +) + +// capabilitySupport is the verdict of one row: whether this client may drive +// this pack at all. +type capabilitySupport string + +const ( + // capabilitySupported — a real client drives this pack on every pull + // request. Confirmed against the conformance workflow, never declared. + capabilitySupported capabilitySupport = "supported" + // capabilityRefused — pointing this client at this pack is stopped before a + // process starts. Confirmed against the pack's own veto. + capabilityRefused capabilitySupport = "refused" +) + +// capabilityProof names the instrument a row rests on. Its value is resolved, +// not printed: capabilityProblems asks the named instrument whether it really +// carries the row. +type capabilityProof string + +const ( + // provenInCI — .github/workflows/conformance.yml runs a suite that drives + // this client against this pack, on every pull request. Read through + // clientsProvenInCI, which is the scan the status table and the client + // version matrix already share. + provenInCI capabilityProof = "conformance workflow" + // refusedAtTheDoorstep — the pack implements packEngineVeto and vetoes this + // engine, so `feint up` and `feint down` stop before starting anything. + refusedAtTheDoorstep capabilityProof = "up.go VetoEngine" +) + +// capabilityControlPlane is the only mode any row can claim today, and the +// column is not decoration: it says what the row's proof covers. The +// conformance matrix starts its emulator with no machine runtime, so every +// proof below is a proof about the control plane — an API that answers, not a +// machine that boots. A row claiming more would need a run under FEINT_VM, and +// modeProblems refuses the whole column the day the workflow arms one without +// this file learning about it. +const capabilityControlPlane = "control plane" + +// capabilityRow is one client against one pack. +type capabilityRow struct { + // Provider is the pack's own name, the spelling clientsProvenInCI keys by + // and Pack.Name() answers. + Provider string + // Client is the token in clientSources: the word prose uses for this + // client, lowercased. + Client string + // Mode says what the proof covers. capabilityControlPlane today, for all of + // them. + Mode string + // Support is the verdict. + Support capabilitySupport + // Reason says why, for a refused row. Empty on a supported one: "it works" + // needs no excuse, and a reason there would be prose nothing checks. + Reason string + // Proof names the instrument that establishes the row. + Proof capabilityProof + // Marker is the token any sentence naming this refused pair must carry, so + // that mentioning the refusal is possible and claiming the capability is + // not. Empty on a supported row. + // + // It is what lets renderPromise write "Terraform joins that pack when a + // release carries exoscale/terraform-provider-exoscale#573" without the + // claim reader treating it as a promise — and what stops "Run your + // Terraform against Scaleway, Outscale or Exoscale" from being written + // again, since that sentence names the pair and carries nothing. + Marker string +} + +// upstreamExoscaleTerraform is the one marker in use, and it is the upstream +// issue itself. +// +// Deliberately the issue and not a word like "refused": a sentence that names +// the pair has to name what would change it, and a reader who meets the marker +// can go and read whether it moved. As of 2026-08-28 the fix is merged into +// upstream `master` (PR #576) and **no release carries it** — the last tag is +// v0.70.0 of 17 July — which is why every sentence generated from this row says +// *until a release carries it* rather than *until it is fixed*. +const upstreamExoscaleTerraform = "exoscale/terraform-provider-exoscale#573" + +// capabilityMatrix is the whole of it, and it is deliberately short: a row per +// pair some instrument establishes, and nothing else. A pair neither the +// workflow drives nor a pack vetoes has no row, so a sentence claiming it is +// refused by unownedCapabilityClaims rather than passing on a row nobody +// earned. +var capabilityMatrix = []capabilityRow{ + {Provider: "scaleway", Client: "terraform", Mode: capabilityControlPlane, Support: capabilitySupported, Proof: provenInCI}, + {Provider: "scaleway", Client: "opentofu", Mode: capabilityControlPlane, Support: capabilitySupported, Proof: provenInCI}, + {Provider: "scaleway", Client: "scw", Mode: capabilityControlPlane, Support: capabilitySupported, Proof: provenInCI}, + + {Provider: "outscale", Client: "terraform", Mode: capabilityControlPlane, Support: capabilitySupported, Proof: provenInCI}, + {Provider: "outscale", Client: "opentofu", Mode: capabilityControlPlane, Support: capabilitySupported, Proof: provenInCI}, + {Provider: "outscale", Client: "octl", Mode: capabilityControlPlane, Support: capabilitySupported, Proof: provenInCI}, + + {Provider: "exoscale", Client: "exo", Mode: capabilityControlPlane, Support: capabilitySupported, Proof: provenInCI}, + { + Provider: "exoscale", Client: "terraform", Mode: capabilityControlPlane, + Support: capabilityRefused, Proof: refusedAtTheDoorstep, Marker: upstreamExoscaleTerraform, + Reason: "the published provider builds two clients and only one honours " + + "`EXOSCALE_API_ENDPOINT`, so an apply or a destroy splits between this emulator and a " + + "paying account (#525 counted five signed requests leaving for `api-ch-*.exoscale.com`). " + + "Upstream exoscale/terraform-provider-exoscale#573 is closed and its fix is merged into " + + "`master`; no published release carries it, the last tag being v0.70.0 of 17 July 2026", + }, + { + Provider: "exoscale", Client: "opentofu", Mode: capabilityControlPlane, + Support: capabilityRefused, Proof: refusedAtTheDoorstep, Marker: upstreamExoscaleTerraform, + Reason: "OpenTofu resolves the same published provider from the same registry namespace, " + + "so it splits the same way and waits on the same release of " + + "exoscale/terraform-provider-exoscale#573", + }, +} + +// capabilityRowFor answers the row for one pair, or nil when nothing +// establishes it. +func capabilityRowFor(provider, client string) *capabilityRow { + for i := range capabilityMatrix { + if capabilityMatrix[i].Provider == provider && capabilityMatrix[i].Client == client { + return &capabilityMatrix[i] + } + } + return nil +} + +// capabilityClientTokens is every client token the matrix and the claim reader +// know, taken from clientSources so the two lists cannot disagree. +func capabilityClientTokens() []string { + out := make([]string, 0, len(clientSources)) + for _, c := range clientSources { + out = append(out, c.token) + } + return out +} + +// capabilityEngineOf answers the `iac.engine` name for a client token, or "" for +// a client that is not an engine `up` runs. +func capabilityEngineOf(token string) string { + for _, c := range clientSources { + if c.token == token { + return c.engine + } + } + return "" +} + +// capabilityClientName spells a token the way the generated tables do. +func capabilityClientName(token string) string { + for _, c := range clientSources { + if c.token == token { + return c.name + } + } + return token +} + +// --------------------------------------------------------------------------- +// Resolving the proofs +// --------------------------------------------------------------------------- + +// packFacts is what the packs themselves say, read once: their names, and which +// engine each one refuses and why. +// +// The packs are asked rather than a table consulted, for the reason #592 is +// about: the refusal that matters is the one `up` and `down` execute, and a +// second declaration of it would be a second thing to disagree with. The names +// come from the same place so that a fourth pack is part of the vocabulary the +// claim reader searches for from the day it mounts a route, without this file +// learning its spelling. +type packFacts struct { + // Providers is every mounted pack's own name. + Providers []string + // Vetoes is provider → engine → the reason the pack gives. + Vetoes map[string]map[string]string +} + +func readPackFacts() (packFacts, error) { + srv, _, err := newServer(nil) + if err != nil { + return packFacts{}, err + } + facts := packFacts{Vetoes: map[string]map[string]string{}} + for _, p := range srv.Packs() { + facts.Providers = append(facts.Providers, p.Name()) + veto, ok := p.(packEngineVeto) + if !ok { + continue + } + // environment.Engines is the list `iac.engine` is validated against, so + // an engine added there is asked about here without this file changing. + for _, engine := range environment.Engines { + if reason := veto.VetoEngine(engine); reason != "" { + if facts.Vetoes[p.Name()] == nil { + facts.Vetoes[p.Name()] = map[string]string{} + } + facts.Vetoes[p.Name()][engine] = reason + } + } + } + sort.Strings(facts.Providers) + return facts, nil +} + +// armedRuntime matches a `--vm` flag with a runtime armed. The mode column +// claims control plane, and it claims it about the workflow's own runs. +var armedRuntime = regexp.MustCompile(`(?m)^[^#\n]*--vm[= ]+([a-z-]+)`) + +// capabilityProblems is what `feint docs` reports in both modes: every +// disagreement between the matrix and the instruments it names. +// +// Reported rather than rendered, on the terms docs.go states for the other +// comparisons: regenerating repairs none of it. A row whose proof does not hold +// needs a decision, not a rewrite. +// +// It answers nothing when the workflow is not there, because `feint docs` also +// regenerates a README outside this repository — the accommodation +// stackProofProblems makes, for the same reason. What stops that from being a +// check that skips itself is TestTheCapabilityChecksHaveASubjectToMeasure, +// which asserts in this repository that the population is not empty and that +// both directions really run. +func capabilityProblems(workflow string) []string { + if _, err := os.Stat(workflow); os.IsNotExist(err) { + return nil + } + facts, err := readPackFacts() + if err != nil { + return []string{fmt.Sprintf("cannot ask the packs what they serve and what they refuse: %v", err)} + } + + var problems []string + problems = append(problems, matrixShapeProblems()...) + problems = append(problems, provenInCIProblems(workflow)...) + problems = append(problems, vetoProblems(facts.Vetoes)...) + problems = append(problems, modeProblems(workflow)...) + sort.Strings(problems) + return problems +} + +// capabilityClaimProblems reads the pages back and reports every claim no +// matrix row carries. +// +// Separate from capabilityProblems above, and the separation is an ordering +// rather than a tidy-up. The checks above judge the *matrix* against the +// instruments and are true of the repository whatever any page says; this one +// judges *what a page says*, so in `--check` mode it reads the documents as +// they stand and in write mode it has to read them **after** the regeneration. +// Run before the writes, it judged the page the run was about to replace, and a +// legitimate change to the matrix could never land: the render was refused +// because of the block it was about to rewrite. +func capabilityClaimProblems(workflow string, pages []string) []string { + if _, err := os.Stat(workflow); os.IsNotExist(err) { + return nil + } + facts, err := readPackFacts() + if err != nil { + return []string{fmt.Sprintf("cannot ask the packs what they serve: %v", err)} + } + var problems []string + for _, page := range pages { + problems = append(problems, unownedCapabilityClaims(page, facts.Providers)...) + } + sort.Strings(problems) + return problems +} + +// matrixShapeProblems refuses a row that cannot be resolved at all: an unknown +// client, an unknown verdict, a refused row with no reason or no marker. +func matrixShapeProblems() []string { + var problems []string + known := map[string]bool{} + for _, token := range capabilityClientTokens() { + known[token] = true + } + // The engine names clientSources carries must be the ones `iac.engine` is + // validated against, or a row would rest on a veto nothing can ever be + // asked for. + for _, c := range clientSources { + if c.engine != "" && !slices.Contains(environment.Engines, c.engine) { + problems = append(problems, fmt.Sprintf( + "clientSources maps %s to the engine %q and internal/environment does not accept it: "+ + "no `feint.yaml` could declare it, so no pack could ever be asked to veto it", + c.name, c.engine)) + } + } + + seen := map[string]bool{} + for _, row := range capabilityMatrix { + key := row.Provider + "/" + row.Client + if seen[key] { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix carries %s twice: two rows about one pair are two answers to one "+ + "question, and capabilityRowFor would only ever read the first", key)) + } + seen[key] = true + if !known[row.Client] { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix names the client %q for %s and clientSources in "+ + "internal/cli/docs_clients.go does not: the claim reader would never look for it", + row.Client, row.Provider)) + } + if row.Mode != capabilityControlPlane { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix says %s is proved in mode %q and this repository can prove only %q: "+ + "a mode nothing runs is a column that reads as a measurement", + key, row.Mode, capabilityControlPlane)) + } + switch row.Support { + case capabilitySupported: + if row.Reason != "" || row.Marker != "" { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix gives %s a reason or a marker and marks it supported: both belong "+ + "to a refusal, and a reader would take them for a caveat on a client that works", key)) + } + case capabilityRefused: + if strings.TrimSpace(row.Reason) == "" { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix refuses %s with no reason: \"refused\" and \"refused because X\" "+ + "are different facts, and only the second is a decision", key)) + } + if strings.TrimSpace(row.Marker) == "" { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix refuses %s with no marker: no sentence could then name the pair "+ + "at all, not even to say it is refused", key)) + } else if !strings.Contains(row.Reason, row.Marker) { + // The published table prints the reason beside the pair, so the + // reason is itself a sentence naming that pair — and it has to + // pass the rule every other sentence passes. Caught by the claim + // reader on its first run against the generated block, which is + // the check finding its own page rather than being told about + // it. + problems = append(problems, fmt.Sprintf( + "capabilityMatrix refuses %s and its reason does not name %s: the published row "+ + "names the pair without naming what would change it, which is the shape this "+ + "whole table refuses everywhere else", key, row.Marker)) + } + default: + problems = append(problems, fmt.Sprintf( + "capabilityMatrix gives %s the verdict %q, which is neither %q nor %q", + key, row.Support, capabilitySupported, capabilityRefused)) + } + } + return problems +} + +// provenInCIProblems resolves every `conformance workflow` proof against the +// workflow, in both directions. +func provenInCIProblems(workflow string) []string { + // proofsPerClient rather than clientsProvenInCI, and the difference is one + // this check was written wrong on first: clientOf maps one suite to the + // cell "Terraform, OpenTofu", and clientsProvenInCI hands that cell back + // verbatim for the status table to print. Compared against a client name it + // matches neither of them, so every engine row read as unproven. The + // transpose splits the cell, which is what a per-client question needs. + proofs, err := proofsPerClient(workflow) + if err != nil { + return []string{fmt.Sprintf("cannot read which clients %s drives: %v", workflow, err)} + } + + // Which pairs the workflow really establishes, as matrix tokens. + drives := map[string]bool{} + for name, list := range proofs { + for _, c := range clientSources { + if c.name != name { + continue + } + for _, proof := range list { + drives[proof.provider+"/"+c.token] = true + } + } + } + + var problems []string + claimed := map[string]bool{} + for _, row := range capabilityMatrix { + if row.Proof != provenInCI { + continue + } + key := row.Provider + "/" + row.Client + claimed[key] = true + if row.Support != capabilitySupported { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix rests %s on %s and does not mark it supported: that proof says a "+ + "client drives the pack, which is the only thing it can say", key, provenInCI)) + continue + } + if !drives[key] { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix says %s is proved by the %s and %s runs no suite driving %s against "+ + "that pack: the row claims a proof that does not exist", + key, provenInCI, workflow, capabilityClientName(row.Client))) + } + } + + // The other direction, and it is the one that keeps the matrix from going + // quiet: a pair CI proves and nobody wrote down is a capability the + // generated sentences may not mention, which understates the product as + // surely as #592 overstated it. + for key := range drives { + if claimed[key] { + continue + } + problems = append(problems, fmt.Sprintf( + "%s drives %s and capabilityMatrix has no row for it: every sentence claiming a client is "+ + "derived from that table, so a proven pair with no row cannot be written down anywhere", + workflow, key)) + } + return problems +} + +// vetoProblems resolves every `up.go VetoEngine` proof against the packs, in +// both directions. +func vetoProblems(vetoes map[string]map[string]string) []string { + var problems []string + claimed := map[string]bool{} + for _, row := range capabilityMatrix { + key := row.Provider + "/" + row.Client + engine := capabilityEngineOf(row.Client) + + if row.Proof == refusedAtTheDoorstep { + claimed[key] = true + if row.Support != capabilityRefused { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix rests %s on %s and does not mark it refused: that proof is a "+ + "refusal and can establish nothing else", key, refusedAtTheDoorstep)) + continue + } + if engine == "" { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix rests %s on %s and %s is not an engine `up` runs: no pack could "+ + "veto it", key, refusedAtTheDoorstep, capabilityClientName(row.Client))) + continue + } + if vetoes[row.Provider][engine] == "" { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix says the %s pack refuses %s at the doorstep and its VetoEngine "+ + "lets that engine through: the table and up.go disagree, which is the defect "+ + "this table exists to make impossible", row.Provider, engine)) + } + continue + } + + // A supported row for an engine a pack really vetoes is #592 exactly, + // written into the table instead of into the README. + if engine != "" && vetoes[row.Provider][engine] != "" { + problems = append(problems, fmt.Sprintf( + "capabilityMatrix marks %s %q and the %s pack vetoes %s before a process starts: "+ + "`feint up` would refuse what the table promises", + key, row.Support, row.Provider, engine)) + } + } + + for provider, engines := range vetoes { + for engine := range engines { + key := provider + "/" + engine + if claimed[key] { + continue + } + problems = append(problems, fmt.Sprintf( + "the %s pack vetoes %s and capabilityMatrix has no row for it: the refusal exists in "+ + "up.go and no generated sentence can say so", provider, engine)) + } + } + return problems +} + +// modeProblems keeps the mode column honest: every row says its proof covers +// the control plane, and the workflow those proofs come from must therefore run +// with no machine runtime armed. +// +// Positive rather than absent: the flag is looked for, and a value other than +// `off` is the finding. A check that passed because it found nothing would be +// the same check whether or not it ever looked. +func modeProblems(workflow string) []string { + body, err := os.ReadFile(workflow) //nolint:gosec // a path this repository owns + if err != nil { + return []string{fmt.Sprintf("cannot read %s to check what the proofs cover: %v", workflow, err)} + } + var problems []string + for _, m := range armedRuntime.FindAllStringSubmatch(string(body), -1) { + if m[1] == "off" { + continue + } + problems = append(problems, fmt.Sprintf( + "%s arms `--vm %s` and every capabilityMatrix row says its proof covers the %q: a run with "+ + "a machine runtime proves more than the column claims, and the column would be measuring "+ + "nothing", workflow, m[1], capabilityControlPlane)) + } + return problems +} diff --git a/internal/cli/capability_claims.go b/internal/cli/capability_claims.go new file mode 100644 index 00000000..0d5b2dc7 --- /dev/null +++ b/internal/cli/capability_claims.go @@ -0,0 +1,250 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "strings" +) + +// Reading the claims a generated block makes, and refusing the ones no matrix +// row carries. +// +// This is the half of #592 that the matrix alone would not deliver. A table +// nothing reads is a table, and the sentence on `README.md:41` was wrong for +// two days precisely because no instrument was pointed at it. So the pages are +// read back: every generated block of the front pages is scanned, every pair of +// (a client this repository knows, a pack this emulator mounts) it asserts is +// looked up in capabilityMatrix, and a pair with no supported row is a problem +// `feint docs --check` exits 2 on. +// +// **What counts as one claim, and the rule is mechanical rather than +// linguistic.** Parsing English is how a control starts lying, so the unit is +// chosen so that co-occurrence is enough: +// +// - a fenced code block is one unit, whole. `eval "$(feint env scaleway)"` +// and `terraform apply` are two lines of one recipe, and the recipe is the +// claim: a quick start showing `feint env exoscale` above `terraform apply` +// is exactly the promise #525 refuses, and it has to redden; +// - a table row is one unit. The generated tables put one client per line, and +// joining them would pair every client with every provider on the page; +// - the rest is prose, split into sentences on `.`, `!` and `?`. A sentence is +// the smallest span in which naming a client and a pack together reads as +// "this drives that". +// +// **And a refusal has to be sayable.** A sentence naming a refused pair passes +// only if it also carries that row's Marker — the upstream issue itself. That +// is what lets the promise block say *"Terraform joins that pack when a release +// carries exoscale/terraform-provider-exoscale#573"* while +// *"Run your Terraform against Scaleway, Outscale or Exoscale"* is refused: the +// second names the pair and carries nothing that would let a reader check. +// +// The reader is falsifiable in both directions, and both are tests: +// TestTheOldFalsePromiseIsCaughtByTheClaimReader plants the exact sentence #592 +// measured and requires it to be reported, and +// TestTheClaimReaderAcceptsWhatTheMatrixCarries requires the sentences this +// repository really ships to pass. A reader that refused everything would pass +// the first alone. + +// capabilityClaimPages are the documents whose generated blocks are read. +// +// The two READMEs because that is where #592 lived, and docs/clients.md because +// it is where the matrix itself is published — a page that prints the table has +// to survive its own rule. +func capabilityClaimPages(root string) []string { + return []string{ + filepath.Join(root, "README.md"), + filepath.Join(root, "README.fr.md"), + filepath.Join(root, "docs", "clients.md"), + } +} + +var ( + // generatedBlock matches one `…` region and + // captures the marker name and the body. Anchored on the marker shape every + // generated section in this repository uses, so a block added later is read + // without this file learning its name. + generatedBlock = regexp.MustCompile(`(?s)(.*?)`) + // wordish is how a unit is tokenised. Lowercase runs of letters and digits, + // so `exoscale/terraform-provider-exoscale#573` yields `terraform` and + // `exoscale` — the pair is named there, and it is the marker that makes the + // sentence legitimate rather than the punctuation hiding it. It also keeps + // `exo` out of `exoscale`, which a substring search would not. + wordish = regexp.MustCompile(`[a-z0-9]+`) + // sentenceEnd splits prose. Deliberately not on `;` or `,`: a semicolon + // joins two independent claims and splitting there would let one sentence + // promise a client for a pack the other excuses. + sentenceEnd = regexp.MustCompile(`[.!?](\s|$)`) + // bulletStart marks a new list item, which starts a new unit. + bulletStart = regexp.MustCompile(`^([-*+]\s|\d+\.\s)`) +) + +// generatedBlockBodies returns the body of every generated section of a +// document, keyed for the message by the marker that names it. +type generatedSection struct { + name string + body string +} + +func generatedBlockBodies(doc string) []generatedSection { + var out []generatedSection + for _, m := range generatedBlock.FindAllStringSubmatch(doc, -1) { + out = append(out, generatedSection{name: m[1], body: m[2]}) + } + return out +} + +// capabilityUnits splits one block into the spans a claim is judged in. +func capabilityUnits(block string) []string { + var units []string + var paragraph []string + var fence []string + inFence := false + + flushParagraph := func() { + if len(paragraph) == 0 { + return + } + text := strings.Join(paragraph, " ") + paragraph = paragraph[:0] + start := 0 + for _, loc := range sentenceEnd.FindAllStringIndex(text, -1) { + units = append(units, text[start:loc[1]]) + start = loc[1] + } + if rest := strings.TrimSpace(text[start:]); rest != "" { + units = append(units, rest) + } + } + + for _, line := range strings.Split(block, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "```") { + if inFence { + units = append(units, strings.Join(fence, "\n")) + fence = fence[:0] + inFence = false + continue + } + flushParagraph() + inFence = true + continue + } + if inFence { + fence = append(fence, line) + continue + } + if trimmed == "" { + flushParagraph() + continue + } + // A table row stands alone: the generated tables put one client on each + // line, and a paragraph made of them would pair every client with every + // provider the table names. + if strings.HasPrefix(trimmed, "|") { + flushParagraph() + units = append(units, trimmed) + continue + } + // So does a list item, for the same reason and with the same + // consequence: the promise block names one pack per item, and two items + // read as one paragraph would claim `scw` for Exoscale. A wrapped + // continuation line carries no bullet marker and joins the item it + // belongs to, which is what the accumulator below does. + if bulletStart.MatchString(trimmed) { + flushParagraph() + } + paragraph = append(paragraph, trimmed) + } + flushParagraph() + // An unterminated fence is content nobody would otherwise read. Keep it + // rather than drop it: dropping is how a reader stops finding things. + if len(fence) > 0 { + units = append(units, strings.Join(fence, "\n")) + } + return units +} + +// capabilityPairsIn answers which (client, provider) pairs one unit names. +func capabilityPairsIn(unit string, providers []string) [][2]string { + words := map[string]bool{} + for _, w := range wordish.FindAllString(strings.ToLower(unit), -1) { + words[w] = true + } + var clients []string + for _, token := range capabilityClientTokens() { + if words[token] { + clients = append(clients, token) + } + } + if len(clients) == 0 { + return nil + } + var pairs [][2]string + for _, provider := range providers { + if !words[provider] { + continue + } + for _, client := range clients { + pairs = append(pairs, [2]string{client, provider}) + } + } + return pairs +} + +// unownedCapabilityClaims reads one page and reports every claim no matrix row +// carries. +// +// A page that is not there is not a finding: `feint docs` regenerates a README +// outside this repository, where docs/clients.md does not exist. What keeps +// that from being a reader that skips itself is +// TestTheCapabilityChecksHaveASubjectToMeasure, which asserts here that the +// pages are read and that they carry blocks. +func unownedCapabilityClaims(path string, providers []string) []string { + body, err := os.ReadFile(path) //nolint:gosec // a path this repository owns + if err != nil { + if os.IsNotExist(err) { + return nil + } + return []string{fmt.Sprintf("cannot read %s to check what it claims: %v", path, err)} + } + + var problems []string + for _, section := range generatedBlockBodies(string(body)) { + for _, unit := range capabilityUnits(section.body) { + for _, pair := range capabilityPairsIn(unit, providers) { + client, provider := pair[0], pair[1] + row := capabilityRowFor(provider, client) + switch { + case row == nil: + problems = append(problems, fmt.Sprintf( + "%s, in the generated `%s` block, puts %s and %s together and capabilityMatrix "+ + "carries no row for that pair: the sentence claims a capability nothing owns, "+ + "which is #592 exactly — %s", + path, section.name, capabilityClientName(client), providerName(provider), + quoteUnit(unit))) + case row.Support == capabilityRefused && + !strings.Contains(strings.ToLower(unit), strings.ToLower(row.Marker)): + problems = append(problems, fmt.Sprintf( + "%s, in the generated `%s` block, puts %s and %s together and capabilityMatrix "+ + "refuses that pair: say why by naming %s, or stop claiming it — %s", + path, section.name, capabilityClientName(client), providerName(provider), + row.Marker, quoteUnit(unit))) + } + } + } + } + return problems +} + +// quoteUnit prints the offending span, shortened, so the message names the +// sentence rather than only the page. +func quoteUnit(unit string) string { + flat := strings.Join(strings.Fields(unit), " ") + const width = 120 + if len(flat) > width { + flat = flat[:width] + "…" + } + return fmt.Sprintf("%q", flat) +} diff --git a/internal/cli/capability_claims_test.go b/internal/cli/capability_claims_test.go new file mode 100644 index 00000000..9cce5b50 --- /dev/null +++ b/internal/cli/capability_claims_test.go @@ -0,0 +1,358 @@ +package cli + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" +) + +// theProviders is the vocabulary the claim reader searches for, taken from the +// packs this binary mounts rather than typed. +func theProviders(t *testing.T) []string { + t.Helper() + facts, err := readPackFacts() + if err != nil { + t.Fatalf("read the pack facts: %v", err) + } + if len(facts.Providers) == 0 { + t.Fatal("no pack is mounted: the claim reader would find no provider name in any sentence") + } + return facts.Providers +} + +// planted writes one generated block to a temporary page and returns what the +// reader says about it. +func planted(t *testing.T, block string) []string { + t.Helper() + page := filepath.Join(t.TempDir(), "README.md") + body := "# feint\n\n\n" + block + "\n\n" + if err := os.WriteFile(page, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return unownedCapabilityClaims(page, theProviders(t)) +} + +// The exact sentence #592 measured, planted, and the reader has to find it. +// +// This is the witness rule: a control whose success is "nothing was found" is +// indistinguishable from a control that looked nowhere. The value planted here +// is not invented — it is `README.md:41` as it stood on `main@3b00d23`, which is +// the sentence that survived every green `docs:check` between #525 landing on +// 2026-08-26 and the audit that read it two days later. +func TestTheOldFalsePromiseIsCaughtByTheClaimReader(t *testing.T) { + const wasOnLine41 = "**Run your Terraform against Scaleway, Outscale or Exoscale without a cloud\n" + + "account, without credentials, and without creating a single resource.**" + + problems := planted(t, wasOnLine41) + if len(problems) == 0 { + t.Fatal("the sentence that was false for two days passed: this reader would have watched " + + "#592 happen and said nothing") + } + joined := strings.Join(problems, "\n") + if !strings.Contains(joined, "Exoscale") || !strings.Contains(joined, "Terraform") { + t.Errorf("the refusal names neither the pack nor the client:\n %s", + strings.Join(problems, "\n ")) + } + // And only that pair. Scaleway and Outscale really are driven by Terraform, + // and a reader that reported them too would be one nobody could act on. + if len(problems) != 1 { + t.Errorf("%d problems for one sentence: the two pairs that are true were reported as "+ + "well, which is a reader that cries at everything\n %s", + len(problems), strings.Join(problems, "\n ")) + } +} + +// And the accepting half, which is the one a refusing-everything reader fails. +func TestTheClaimReaderAcceptsWhatTheMatrixCarries(t *testing.T) { + for _, accepted := range []struct { + name string + block string + }{ + { + "the promise this repository ships", + mustRenderPromise(t, false), + }, + { + "the French promise", + mustRenderPromise(t, true), + }, + { + "a recipe pointing Terraform at Scaleway", + "```bash\nfeint start\neval \"$(feint env scaleway)\"\nterraform apply\n```", + }, + { + "a sentence naming the refused pair and the reason", + "Terraform joins Exoscale the day a release carries " + + "exoscale/terraform-provider-exoscale#573.", + }, + { + "a table row per client", + "| Client | Drives |\n|---|---|\n| Terraform | Scaleway, Outscale |\n| `exo` | Exoscale |", + }, + } { + t.Run(accepted.name, func(t *testing.T) { + if problems := planted(t, accepted.block); len(problems) != 0 { + t.Errorf("a legitimate block was refused, so the reader breaks the product it "+ + "protects:\n %s", strings.Join(problems, "\n ")) + } + }) + } +} + +// And the refusing half, on the shapes the unit rule exists for. +func TestTheClaimReaderRefusesEveryShapeOfTheClaim(t *testing.T) { + for _, refused := range []struct { + name string + block string + want string + }{ + { + "a quick start pointing Terraform at the pack the doorstep refuses", + "```bash\nfeint start\neval \"$(feint env exoscale)\"\nterraform apply\n```", + "Exoscale", + }, + { + "a sentence naming the refused pair without saying why", + "Terraform drives Exoscale as well.", + "exoscale/terraform-provider-exoscale#573", + }, + { + "a table row claiming it", + "| Client | Drives |\n|---|---|\n| Terraform | Exoscale |", + "Exoscale", + }, + { + "a list item claiming it", + "- **Exoscale** with Terraform.\n- **Scaleway** with `scw`.", + "Exoscale", + }, + { + "a pair no row carries at all", + "The `scw` CLI drives Outscale.", + "carries no row", + }, + } { + t.Run(refused.name, func(t *testing.T) { + problems := planted(t, refused.block) + if len(problems) == 0 { + t.Fatalf("this block claimed a capability nothing owns and passed:\n%s", refused.block) + } + if !strings.Contains(strings.Join(problems, "\n"), refused.want) { + t.Errorf("the refusal does not say what to fix:\n %s", strings.Join(problems, "\n ")) + } + }) + } +} + +// The unit rule is what makes co-occurrence mean something, and each of its +// three cases was needed by a real block in this repository. +func TestAUnitIsASentenceATableRowAListItemOrAWholeFence(t *testing.T) { + block := "" + + "| Terraform | Outscale, Scaleway |\n" + + "| `exo` | Exoscale |\n" + + "\n" + + "First sentence. Second sentence.\n" + + "\n" + + "- one item\n" + + "- another item\n" + + "\n" + + "```bash\nline one\nline two\n```\n" + + units := capabilityUnits(block) + want := []string{ + "| Terraform | Outscale, Scaleway |", + "| `exo` | Exoscale |", + "First sentence.", + "Second sentence.", + "- one item", + "- another item", + "line one\nline two", + } + if len(units) != len(want) { + t.Fatalf("got %d units, want %d:\n %s", len(units), len(want), strings.Join(units, "\n ")) + } + for i := range want { + if strings.TrimSpace(units[i]) != want[i] { + t.Errorf("unit %d is %q, want %q", i, strings.TrimSpace(units[i]), want[i]) + } + } +} + +// `exo` is not hiding inside `exoscale`, and `terraform` really is inside +// `terraform-provider-exoscale`. +// +// Both matter and they pull in opposite directions. A substring search would +// find `exo` in every mention of the pack and claim a CLI nobody named; a +// whole-token rule that stopped at punctuation would miss the client named in +// the upstream issue reference, which is exactly the string a legitimate +// sentence about the refusal carries. +func TestTheReaderTokenisesRatherThanSearchesForSubstrings(t *testing.T) { + providers := theProviders(t) + + if pairs := capabilityPairsIn("Exoscale is a cloud.", providers); len(pairs) != 0 { + t.Errorf("`exo` was found inside `Exoscale`: %v", pairs) + } + pairs := capabilityPairsIn("exoscale/terraform-provider-exoscale#573", providers) + found := false + for _, pair := range pairs { + if pair[0] == "terraform" && pair[1] == "exoscale" { + found = true + } + } + if !found { + t.Errorf("the upstream reference names both and the reader saw neither: %v", pairs) + } +} + +// A configuration whose resource blocks are not its objects is refused rather +// than counted wrong. +// +// The equality "one `resource` block, one object" is what lets the front page +// print a number, and `count` or `for_each` breaks it silently — the figure +// would be off by however many the loop makes, on the one line a first-time +// reader checks against their own terminal. Refusing says the example has +// outgrown the job; counting anyway would publish an invented measurement. +func TestAQuickStartThatMultipliesItsResourcesIsRefusedRatherThanMiscounted(t *testing.T) { + dir := t.TempDir() + write := func(body string) string { + t.Helper() + path := filepath.Join(dir, "main.tf") + if err := os.WriteFile(path, []byte(body), 0o600); err != nil { + t.Fatal(err) + } + return path + } + + // The accepting half first: a plain configuration counts. + plain := write("resource \"a\" \"one\" {}\nresource \"b\" \"two\" {\n name = \"x\"\n}\n" + + "# resource \"c\" \"commented\" {}\n") + got, err := resourceCount(plain) + if err != nil { + t.Fatalf("a plain configuration was refused: %v", err) + } + if got != 2 { + t.Errorf("counted %d resources, want 2: a commented block or a nested one was miscounted", got) + } + + for _, refused := range []struct{ name, body string }{ + {"count", "resource \"a\" \"one\" {\n count = 3\n}\n"}, + {"for_each", "resource \"a\" \"one\" {\n for_each = toset([\"x\", \"y\"])\n}\n"}, + {"nothing at all", "variable \"endpoint\" {\n type = string\n}\n"}, + } { + t.Run(refused.name, func(t *testing.T) { + if _, err := resourceCount(write(refused.body)); err == nil { + t.Errorf("a configuration whose blocks are not its objects was counted anyway, so "+ + "the README would print a number nobody produces (%s)", refused.name) + } + }) + } +} + +// mustRenderPromise is the block this repository really ships, in one locale. +func mustRenderPromise(t *testing.T, french bool) string { + t.Helper() + rendered, err := renderPromise(french) + if err != nil { + t.Fatalf("render the promise: %v", err) + } + return rendered +} + +// The French page is in French. +// +// #591's fourth finding: the generated blocks were injected in English into both +// READMEs on the rule that *a command needs no translation*, and the rule was +// right about commands and wrong about the prose beside them — README.fr.md +// carried "On your machine" and "In CI, or anywhere Docker runs" in the middle +// of a French page. +func TestTheFrenchQuickStartIsInFrench(t *testing.T) { + root := repoRoot(t) + rendered, err := renderQuickstart(filepath.Join(root, goModPath), filepath.Join(root, changelogPath), true) + if err != nil { + t.Fatalf("render the French quick start: %v", err) + } + // The sentences that were landing in English, named one by one so a + // regression says which. + for _, english := range []string{ + "On your machine", + "In CI, or anywhere Docker runs", + "In GitHub Actions without a container", + "Already have a project?", + "detaches, waits until it answers", + } { + if strings.Contains(rendered, english) { + t.Errorf("the French quick start carries %q", english) + } + } + // And the accepting half: it really is the block, not an empty string that + // would pass every assertion above. + for _, french := range []string{ + "Sur votre machine", + "En CI, ou partout où Docker tourne", + "Vous avez déjà un projet ?", + } { + if !strings.Contains(rendered, french) { + t.Errorf("the French quick start does not carry %q, so the check above measured an "+ + "empty render", french) + } + } + // The commands themselves are not translated, which is the half of the old + // rule that was right. + for _, command := range []string{"feint up", "feint down", "terraform apply", "feint start"} { + if !strings.Contains(rendered, command) { + t.Errorf("the French quick start lost the command %q", command) + } + } +} + +// The quick start sends a reader at a directory that exists and that a suite +// applies. +// +// #593's complaint in one line: the block printed `terraform apply` with no +// directory, no `main.tf` and no provider block, under an `Apply complete!` that +// was not reachable from it. A path in a generated block that resolves to +// nothing is the same defect with a longer walk. +func TestTheQuickStartPointsAtADirectoryThatExists(t *testing.T) { + root := repoRoot(t) + rendered, err := renderQuickstart(filepath.Join(root, goModPath), filepath.Join(root, changelogPath), false) + if err != nil { + t.Fatalf("render the quick start: %v", err) + } + stack := quickstartRoot + "/" + quickstartLead + if !strings.Contains(rendered, stack) { + t.Fatalf("the quick start does not name %s:\n%s", stack, rendered) + } + for _, needed := range []string{"main.tf", "feint.yaml"} { + path := filepath.Join(root, quickstartRoot, quickstartLead, needed) + if _, err := os.Stat(path); err != nil { + t.Errorf("the quick start sends a reader to %s and it has no %s: the four commands "+ + "cannot work", stack, needed) + } + } + // The output it displays is the output those commands produce, counted from + // the configuration rather than typed. `Resources: 5 added` sat under three + // commands that could not produce it, and that is #593's second finding. + added, err := resourceCount(filepath.Join(root, quickstartRoot, quickstartLead, "main.tf")) + if err != nil { + t.Fatalf("count what the quick start creates: %v", err) + } + line := fmt.Sprintf("Apply complete! Resources: %d added, 0 changed, 0 destroyed.", added) + if !strings.Contains(rendered, line) { + t.Errorf("the quick start does not print %q, so the output it shows is not derived from "+ + "the configuration it runs", line) + } + + // And it teaches the verbs this project ships rather than the ones before + // them: `feint up` first, the hand-driven form underneath. + up := strings.Index(rendered, "feint up") + start := strings.Index(rendered, "feint start") + if up < 0 || start < 0 { + t.Fatalf("the quick start lost one of the two doors:\n%s", rendered) + } + if up > start { + t.Error("the quick start teaches `feint start` before `feint up`, which is the 0.10 " + + "sequence in front of the one 0.11 ships (#593)") + } +} diff --git a/internal/cli/capability_render.go b/internal/cli/capability_render.go new file mode 100644 index 00000000..425b3c09 --- /dev/null +++ b/internal/cli/capability_render.go @@ -0,0 +1,357 @@ +package cli + +import ( + "fmt" + "os" + "sort" + "strings" +) + +// The sentences derived from capabilityMatrix, and the table that publishes it. +// +// Two consumers, and they are the two halves of #592. The promise block is the +// README's own line 41, generated in both locales so that the claim is no +// longer something anybody types; the capability block in docs/clients.md is +// the matrix itself, printed with its proof column so a reader can check the +// promise against what establishes it. +// +// The prose is written per locale rather than shared, and that is #591's fourth +// finding rather than a preference: the generated blocks used to be injected in +// English into both pages, on the rule that *a command needs no translation*. +// True of a command; the blocks also carry prose, and README.fr.md carried +// "On your machine" and "In CI, or anywhere Docker runs" in the middle of a +// French page. So what is shared is what is structured — the matrix, the +// version, the image, the commands — and what is written twice is the sentence +// around them. + +// supportedPairs answers, per client token, the packs the matrix marks +// supported — the raw material every promise sentence is built from. +func supportedPairs() map[string][]string { + out := map[string][]string{} + for _, row := range capabilityMatrix { + if row.Support != capabilitySupported { + continue + } + out[row.Client] = append(out[row.Client], row.Provider) + } + return out +} + +// refusedRows answers the refused rows in the order capabilityMatrix declares +// them. +// +// Declaration order rather than alphabetical, here and in supportedPairs above, +// and the prose is why: sorted, the first sentence of the README read "OpenTofu +// and Terraform drive Outscale and Scaleway", which is every name in the right +// place and the emphasis in the wrong one. The order a maintainer writes the +// rows in is a judgement about which pack and which client a reader meets +// first, and it is worth keeping. +func refusedRows() []capabilityRow { + var out []capabilityRow + for _, row := range capabilityMatrix { + if row.Support == capabilityRefused { + out = append(out, row) + } + } + return out +} + +// joinNames renders a list of names the way the locale writes one. +func joinNames(names []string, and string) string { + switch len(names) { + case 0: + return "" + case 1: + return names[0] + default: + return strings.Join(names[:len(names)-1], ", ") + " " + and + " " + names[len(names)-1] + } +} + +// engineProviders answers the packs every IaC engine drives, and the packs +// where at least one engine is refused. +// +// It insists the engines agree. Terraform and OpenTofu resolve the same +// providers from the same registry, so a promise that lumped them together +// while the matrix split them would be a sentence truer than the table. If they +// ever diverge, the render fails rather than picking one. +func engineProviders() (driven []string, refused []capabilityRow, err error) { + supported := supportedPairs() + var engines []string + for _, c := range clientSources { + if c.engine != "" { + engines = append(engines, c.token) + } + } + if len(engines) == 0 { + return nil, nil, fmt.Errorf("clientSources names no infrastructure-as-code engine: " + + "the promise has no client to promise") + } + driven = supported[engines[0]] + for _, engine := range engines[1:] { + if strings.Join(supported[engine], ",") != strings.Join(driven, ",") { + return nil, nil, fmt.Errorf( + "capabilityMatrix has %s driving %v and %s driving %v: the promise names one engine "+ + "list for both, and it cannot while they disagree", + capabilityClientName(engines[0]), driven, capabilityClientName(engine), supported[engine]) + } + } + for _, row := range refusedRows() { + if capabilityEngineOf(row.Client) != "" { + refused = append(refused, row) + } + } + return driven, refused, nil +} + +// renderPromise writes the README's promise: what this emulator lets a reader +// do, and for which pack with which client. +// +// Every name in it comes from capabilityMatrix. The sentence that used to sit +// here said "Run your Terraform against Scaleway, Outscale or Exoscale" and was +// false for two days; it cannot be written again, because nobody writes this +// paragraph. +// The shape is a sentence and a list rather than one paragraph, and that is the +// claim reader's doing rather than a layout choice: a unit naming two packs and +// two clients asserts every pair of them, so a single sentence reading "`scw` +// for Scaleway, `octl` for Outscale, `exo` for Exoscale" would be claiming +// `scw` against Exoscale. One pack per list item is what makes each line a +// claim the matrix can carry, and it is the honest shape anyway — the packs do +// not have the same clients. +func renderPromise(french bool) (string, error) { + driven, refused, err := engineProviders() + if err != nil { + return "", err + } + + var engineNames, drivenNames []string + for _, c := range clientSources { + if c.engine != "" { + engineNames = append(engineNames, c.name) + } + } + for _, provider := range driven { + drivenNames = append(drivenNames, providerName(provider)) + } + + // The refusals a given pack carries, so the list item that names the pack + // also names the way on — the rule VetoEngine states about its own reason. + refusedBy := map[string][]capabilityRow{} + for _, row := range refused { + refusedBy[row.Provider] = append(refusedBy[row.Provider], row) + } + + var b strings.Builder + b.WriteString(docsGenerated) + b.WriteString("\n\n") + + if french { + b.WriteString("**Pointez Terraform et les CLI officielles vers votre propre machine.**\n") + b.WriteString("Aucun compte cloud, aucun identifiant, et rien de créé nulle part.\n\n") + b.WriteString(wrapParagraph(fmt.Sprintf( + "%s pilotent %s. Chaque pack a en plus son CLI officiel, et chacun d'eux pilote cet "+ + "émulateur de bout en bout :", + joinNames(engineNames, "et"), joinNames(drivenNames, "et")))) + b.WriteString("\n") + } else { + b.WriteString("**Point Terraform and the official cloud CLIs at your own machine.**\n") + b.WriteString("No cloud account, no credentials, and nothing created anywhere.\n\n") + b.WriteString(wrapParagraph(fmt.Sprintf( + "%s drive %s. Each pack also has its own official CLI, and every one of them drives "+ + "this emulator end to end:", + joinNames(engineNames, "and"), joinNames(drivenNames, "and")))) + b.WriteString("\n") + } + + for _, row := range cliRows() { + item := fmt.Sprintf("**%s** with `%s`", providerName(row.Provider), row.Client) + if french { + item = fmt.Sprintf("**%s** avec `%s`", providerName(row.Provider), row.Client) + } + // Every engine the pack refuses, not the first one: Terraform and + // OpenTofu are refused for the same reason and carry the same marker, + // and naming one of them would read as the other being allowed. + if refusals := refusedBy[row.Provider]; len(refusals) > 0 { + var names []string + for _, refusal := range refusals { + names = append(names, capabilityClientName(refusal.Client)) + } + // A second sentence rather than a clause, and the claim reader is + // why it may be one: it names the pack only through the marker, and + // the marker is what makes naming a refused pair legitimate. It + // also avoids "et Terraform et OpenTofu" in the French. + if french { + item += fmt.Sprintf( + ". %s reviennent le jour où une version publiée porte le correctif de %s, "+ + "que `feint up` refuse au portillon jusque-là", + joinNames(names, "et"), refusals[0].Marker) + } else { + item += fmt.Sprintf( + ". %s join it the day a published release carries the fix for %s, which "+ + "`feint up` refuses at the doorstep until one does", + joinNames(names, "and"), refusals[0].Marker) + } + } + b.WriteString(wrapBullet(item + ".")) + } + return b.String(), nil +} + +// cliRows are the supported rows for clients that are not engines: one per +// pack, in the order clientSources declares them. +func cliRows() []capabilityRow { + var out []capabilityRow + for _, c := range clientSources { + if c.engine != "" { + continue + } + for _, row := range capabilityMatrix { + if row.Client == c.token && row.Support == capabilitySupported { + out = append(out, row) + } + } + } + return out +} + +// renderCapabilityMatrix publishes the table itself, proof column included. +// +// The proof column is the reason this is more than a list. `supported` names +// the workflow that drives the pair on every pull request; `refused` names the +// doorstep that stops it. Neither is typed here: capabilityProblems resolves +// both against those instruments, and `feint docs --check` exits 2 when one of +// them stops carrying its row. +func renderCapabilityMatrix() string { + var b strings.Builder + b.WriteString(docsGenerated) + b.WriteString("\n\n") + b.WriteString("| Provider | Client | Mode | Support | Proof | Reason |\n|---|---|---|---|---|---|\n") + + rows := append([]capabilityRow{}, capabilityMatrix...) + sort.SliceStable(rows, func(i, j int) bool { + if rows[i].Provider != rows[j].Provider { + return rows[i].Provider < rows[j].Provider + } + return rows[i].Client < rows[j].Client + }) + for _, row := range rows { + reason := row.Reason + if reason == "" { + reason = "—" + } + fmt.Fprintf(&b, "| %s | %s | %s | **%s** | `%s` | %s |\n", + providerName(row.Provider), capabilityClientName(row.Client), row.Mode, + row.Support, row.Proof, reason) + } + + b.WriteString("\nEvery sentence in this repository that claims a client for a provider is\n") + b.WriteString("generated from this table, and every row is resolved against the thing its\n") + b.WriteString("proof column names: `conformance workflow` means\n") + b.WriteString("`.github/workflows/conformance.yml` runs a suite driving that client against\n") + b.WriteString("that pack on every pull request, and `up.go VetoEngine` means the pack itself\n") + b.WriteString("refuses the engine before `feint up` starts a process. A row whose proof does\n") + b.WriteString("not carry it fails `feint docs --check` at exit code 2, and so does a page\n") + b.WriteString("that claims a pair this table does not.\n") + return b.String() +} + +// --------------------------------------------------------------------------- +// Splicing +// --------------------------------------------------------------------------- + +// splicePromise reports whether either front page's promise is out of date. +func splicePromise(root string) (bool, error) { + changed := false + for _, page := range frontPages(root) { + updated, current, err := promiseFor(page) + if err != nil { + return false, err + } + if current != "" && updated != current { + changed = true + } + } + return changed, nil +} + +func writeSplicedPromise(root string) error { + for _, page := range frontPages(root) { + updated, current, err := promiseFor(page) + if err != nil { + return err + } + if current == "" || updated == current { + continue + } + if err := os.WriteFile(page.Path, []byte(updated), 0o644); err != nil { //nolint:gosec // documentation is world-readable by design + return err + } + } + return nil +} + +// promiseFor renders one page's promise and returns the updated document beside +// the current one. An empty `current` means the page is absent or claims no +// promise block, and the caller does nothing. +func promiseFor(page frontPage) (updated, current string, err error) { + body, err := os.ReadFile(page.Path) //nolint:gosec // a path this repository owns + if err != nil { + if os.IsNotExist(err) { + return "", "", nil + } + return "", "", err + } + if !strings.Contains(string(body), promiseStartMarker) { + return "", "", nil + } + rendered, err := renderPromise(page.French) + if err != nil { + return "", "", err + } + out, err := spliceSection(string(body), promiseStartMarker, promiseEndMarker, rendered) + if err != nil { + return "", "", fmt.Errorf("%s: %w", page.Path, err) + } + return out, string(body), nil +} + +// spliceCapability keeps the published matrix in step with the declared one. +func spliceCapability(path string) (bool, error) { + updated, current, err := capabilityFor(path) + if err != nil { + return false, err + } + return current != "" && updated != current, nil +} + +func writeSplicedCapability(path string) error { + updated, current, err := capabilityFor(path) + if err != nil { + return err + } + if current == "" || updated == current { + return nil + } + return os.WriteFile(path, []byte(updated), 0o644) //nolint:gosec // documentation is world-readable by design +} + +func capabilityFor(path string) (updated, current string, err error) { + if path == "" { + return "", "", nil + } + body, err := os.ReadFile(path) //nolint:gosec // a path this repository owns + if err != nil { + if os.IsNotExist(err) { + return "", "", nil + } + return "", "", err + } + if !strings.Contains(string(body), capabilityStartMarker) { + return "", "", nil + } + out, err := spliceSection(string(body), capabilityStartMarker, capabilityEndMarker, renderCapabilityMatrix()) + if err != nil { + return "", "", fmt.Errorf("%s: %w", path, err) + } + return out, string(body), nil +} diff --git a/internal/cli/capability_test.go b/internal/cli/capability_test.go new file mode 100644 index 00000000..22daa2cd --- /dev/null +++ b/internal/cli/capability_test.go @@ -0,0 +1,314 @@ +package cli + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stephrobert/feint/internal/environment" +) + +// The repository satisfies its own capability rule. +// +// The accepting half, and it belongs first: a matrix that refused everything +// would pass every mutation below and break the product. +func TestTheCapabilityMatrixHoldsAgainstItsOwnInstruments(t *testing.T) { + root := repoRoot(t) + workflow := filepath.Join(root, conformanceWorkflow) + if problems := capabilityProblems(workflow); len(problems) != 0 { + t.Fatalf("the repository does not satisfy its own rule:\n %s", strings.Join(problems, "\n ")) + } + if problems := capabilityClaimProblems(workflow, capabilityClaimPages(root)); len(problems) != 0 { + t.Fatalf("a generated block claims something no matrix row carries:\n %s", + strings.Join(problems, "\n ")) + } +} + +// The population is not empty, and both directions of both proofs really run. +// +// capabilityProblems answers nothing when the workflow is absent, because +// `feint docs` also regenerates a README outside this repository. That +// tolerance is exactly the shape of a check that stops measuring when its +// subject moves, so the subject is asserted here: there are rows, both verdicts +// occur, both proofs occur, and the pages the claim reader walks really carry +// generated blocks. +func TestTheCapabilityChecksHaveASubjectToMeasure(t *testing.T) { + root := repoRoot(t) + + var supported, refused, byCI, byVeto int + for _, row := range capabilityMatrix { + switch row.Support { + case capabilitySupported: + supported++ + case capabilityRefused: + refused++ + } + switch row.Proof { + case provenInCI: + byCI++ + case refusedAtTheDoorstep: + byVeto++ + } + } + if supported == 0 || refused == 0 { + t.Fatalf("%d supported and %d refused rows: a matrix with one verdict measures neither", + supported, refused) + } + if byCI == 0 || byVeto == 0 { + t.Fatalf("%d rows proved by CI and %d by the doorstep: a proof nothing rests on is a "+ + "resolver nobody exercises", byCI, byVeto) + } + + // And the claim reader has something to read. A page with no generated + // block passes every claim rule while reading nothing. + for _, page := range capabilityClaimPages(root) { + body, err := os.ReadFile(page) + if err != nil { + t.Fatalf("read %s: %v", page, err) + } + blocks := generatedBlockBodies(string(body)) + if len(blocks) == 0 { + t.Errorf("%s carries no generated block: the claim reader walks it and finds nothing "+ + "to judge", page) + } + units := 0 + for _, block := range blocks { + units += len(capabilityUnits(block.body)) + } + if units == 0 { + t.Errorf("%s splits into no unit at all: the reader would report nothing whatever the "+ + "page said", page) + } + } +} + +// The matrix and the doorstep cannot disagree, in either direction. +// +// This is the property #592 asks for in one sentence: *a refusal that lives in +// up.go and a table that disagree is the same defect one level down.* Both +// mutations are the ones somebody will really make — a row flipped to +// supported, and a veto the table never learned about. +func TestTheMatrixAndTheDoorstepCannotDisagree(t *testing.T) { + facts, err := readPackFacts() + if err != nil { + t.Fatalf("read the pack facts: %v", err) + } + if len(facts.Vetoes) == 0 { + t.Fatal("no pack vetoes any engine: the doorstep of #525 is disarmed, and every assertion " + + "below would pass by having nothing to compare") + } + if problems := vetoProblems(facts.Vetoes); len(problems) != 0 { + t.Fatalf("the matrix and the packs already disagree:\n %s", strings.Join(problems, "\n ")) + } + + // A row that promises what the doorstep refuses — #592 written into the + // table rather than into the README. + restore := capabilityMatrix + t.Cleanup(func() { capabilityMatrix = restore }) + + flipped := append([]capabilityRow{}, restore...) + found := false + for i := range flipped { + if flipped[i].Support != capabilityRefused { + continue + } + flipped[i].Support = capabilitySupported + flipped[i].Proof = provenInCI + found = true + } + if !found { + t.Fatal("no refused row to flip: this test is measuring a table it does not understand") + } + capabilityMatrix = flipped + // Asserted on what the refusal *says*, not on there being one, and the + // falsification is why. With the flipped-row guard neutralised, the other + // rule below still fires about the same matrix — the pair is vetoed and no + // row claims the doorstep — so a test reading only "something was reported" + // stayed green through a mutation that had disarmed exactly the guard it + // names. Two correct findings about one mutated table are not + // interchangeable. + flippedProblems := strings.Join(vetoProblems(facts.Vetoes), "\n") + if !strings.Contains(flippedProblems, "before a process starts") { + t.Errorf("a row promising a client the pack vetoes passed: `feint up` would refuse what "+ + "the README promises, which is the defect this table exists to make impossible\n %s", + flippedProblems) + } + + // And the other direction: a veto the table never learned about. Dropping + // the rows rather than editing them, because the silent failure is the one + // where nothing is written down at all. + var withoutRefusals []capabilityRow + for _, row := range restore { + if row.Support != capabilityRefused { + withoutRefusals = append(withoutRefusals, row) + } + } + capabilityMatrix = withoutRefusals + problems := vetoProblems(facts.Vetoes) + if len(problems) == 0 { + t.Error("a pack vetoes an engine and no row says so, and it passed: the refusal would " + + "exist in up.go with no generated sentence able to mention it") + } + if !strings.Contains(strings.Join(problems, "\n"), "exoscale") { + t.Errorf("the refusal names no pack, which is the one thing needed to fix it:\n %s", + strings.Join(problems, "\n ")) + } +} + +// A supported row rests on a workflow that really drives that pair, and a pair +// CI drives has a row. +// +// The second direction is the understated half, and this repository has paid +// for it once already: an external review recommended deleting Terraform from +// the README's Outscale row on the strength of a table that had understated it, +// which would have erased a suite applying twenty-one resources. +func TestASupportedRowNamesAWorkflowThatDrivesIt(t *testing.T) { + root := repoRoot(t) + workflow := filepath.Join(root, conformanceWorkflow) + if problems := provenInCIProblems(workflow); len(problems) != 0 { + t.Fatalf("the repository does not satisfy its own rule:\n %s", strings.Join(problems, "\n ")) + } + + restore := capabilityMatrix + t.Cleanup(func() { capabilityMatrix = restore }) + + // A pair nothing drives, claiming the workflow proves it. + capabilityMatrix = append(append([]capabilityRow{}, restore...), capabilityRow{ + Provider: "exoscale", Client: "scw", Mode: capabilityControlPlane, + Support: capabilitySupported, Proof: provenInCI, + }) + problems := provenInCIProblems(workflow) + if len(problems) == 0 { + t.Error("a row claiming the conformance workflow drives `scw` against Exoscale passed: " + + "the proof column would be decoration") + } + + // And a pair the workflow drives with no row at all. + var missing []capabilityRow + dropped := "" + for _, row := range restore { + if row.Proof == provenInCI && dropped == "" { + dropped = row.Provider + "/" + row.Client + continue + } + missing = append(missing, row) + } + if dropped == "" { + t.Fatal("no row rests on the workflow: this test is measuring a table it does not understand") + } + capabilityMatrix = missing + problems = provenInCIProblems(workflow) + if len(problems) == 0 { + t.Errorf("%s is driven on every pull request and no row carries it, and it passed: no "+ + "generated sentence could ever mention a client this project really proves", dropped) + } +} + +// The mode column says what the proofs cover, and it is read from the workflow +// rather than asserted. +// +// Every row claims the control plane, because the conformance matrix starts its +// emulator with no machine runtime. The mutation is the one that would make the +// column a lie: a leg that arms one. +func TestTheModeColumnIsReadFromTheRunItDescribes(t *testing.T) { + root := repoRoot(t) + workflow := filepath.Join(root, conformanceWorkflow) + if problems := modeProblems(workflow); len(problems) != 0 { + t.Fatalf("the conformance workflow already arms a runtime:\n %s", strings.Join(problems, "\n ")) + } + + body, err := os.ReadFile(workflow) + if err != nil { + t.Fatal(err) + } + armed := strings.Replace(string(body), + "/tmp/feint start --addr 127.0.0.1:4599", + "/tmp/feint start --vm incus-ovn --addr 127.0.0.1:4599", 1) + if armed == string(body) { + t.Fatal("the workflow no longer starts the emulator the way this test expects: it is " + + "measuring a file it does not understand") + } + copied := filepath.Join(t.TempDir(), "conformance.yml") + if err := os.WriteFile(copied, []byte(armed), 0o600); err != nil { + t.Fatal(err) + } + problems := modeProblems(copied) + if len(problems) == 0 { + t.Error("a workflow arming `--vm incus-ovn` left every row claiming the control plane " + + "unchallenged: the column would describe a run that no longer exists") + } +} + +// A refused row is a decision, and a decision has a reason and a way to check +// whether it still holds. +func TestARefusedRowCarriesItsReasonAndItsMarker(t *testing.T) { + if problems := matrixShapeProblems(); len(problems) != 0 { + t.Fatalf("the matrix does not satisfy its own shape rule:\n %s", strings.Join(problems, "\n ")) + } + + restore := capabilityMatrix + t.Cleanup(func() { capabilityMatrix = restore }) + + for _, mutation := range []struct { + name string + apply func(row *capabilityRow) + want string + }{ + {"no reason", func(row *capabilityRow) { row.Reason = " " }, "no reason"}, + {"no marker", func(row *capabilityRow) { row.Marker = "" }, "no marker"}, + {"a reason that does not name the marker", + func(row *capabilityRow) { row.Reason = "it splits between two clouds" }, "does not name"}, + {"an unknown mode", func(row *capabilityRow) { row.Mode = "with a machine runtime" }, "mode"}, + } { + t.Run(mutation.name, func(t *testing.T) { + mutated := append([]capabilityRow{}, restore...) + touched := false + for i := range mutated { + if mutated[i].Support == capabilityRefused { + mutation.apply(&mutated[i]) + touched = true + break + } + } + if !touched { + t.Fatal("no refused row to mutate") + } + capabilityMatrix = mutated + problems := matrixShapeProblems() + if len(problems) == 0 { + t.Fatalf("a refused row with %s passed", mutation.name) + } + if !strings.Contains(strings.Join(problems, "\n"), mutation.want) { + t.Errorf("the refusal does not say what is wrong:\n %s", strings.Join(problems, "\n ")) + } + }) + } +} + +// Every engine clientSources names is an engine `feint.yaml` accepts. +// +// A row resting on a veto for an engine internal/environment refuses could +// never be asked for, so the proof would be unfalsifiable rather than true. +func TestEveryEngineTheMatrixKnowsIsOneUpCanBeAskedToRun(t *testing.T) { + seen := 0 + for _, c := range clientSources { + if c.engine == "" { + continue + } + seen++ + found := false + for _, engine := range environment.Engines { + if engine == c.engine { + found = true + } + } + if !found { + t.Errorf("clientSources maps %s to the engine %q and internal/environment does not "+ + "accept it", c.name, c.engine) + } + } + if seen == 0 { + t.Fatal("clientSources names no engine at all: every engine assertion here measures nothing") + } +} diff --git a/internal/cli/docs.go b/internal/cli/docs.go index a629b354..540e8f42 100644 --- a/internal/cli/docs.go +++ b/internal/cli/docs.go @@ -277,7 +277,7 @@ func docs(args []string, stdout, stderr io.Writer) int { // workflow that installs the clients and every required_providers block the // fixtures and stacks declare, so a release cannot claim a client version // that nothing ran, nor a provider version that nothing pinned. - provedChanged, provedErr := spliceProved(*clientsPage, *workflow, conformanceRoot, stacksRoot, stacksScript) + provedChanged, provedErr := spliceProved(*clientsPage, *workflow, conformanceRoot, exampleSources()) if provedErr != nil { fmt.Fprintf(stderr, "feint: %v\n", provedErr) return exitError @@ -293,6 +293,24 @@ func docs(args []string, stdout, stderr io.Writer) int { return exitError } + // The README's promise, in both locales, derived from capabilityMatrix + // (#592). The sentence it replaced claimed Terraform on three packs while + // `feint up` refused one of them, and survived every green `docs:check` + // between #525 and the audit that read it — because it was prose, and prose + // is what the rest of this chain cannot own. + promiseChanged, promiseErr := splicePromise(filepath.Dir(*target)) + if promiseErr != nil { + fmt.Fprintf(stderr, "feint: %v\n", promiseErr) + return exitError + } + // And the matrix itself, published beside the client versions it shares a + // source with. + capabilityChanged, capabilityErr := spliceCapability(*clientsPage) + if capabilityErr != nil { + fmt.Fprintf(stderr, "feint: %v\n", capabilityErr) + return exitError + } + installChanged, installErr := splicePrereq(*installDoc, goModPath) if installErr != nil { fmt.Fprintf(stderr, "feint: %v\n", installErr) @@ -321,13 +339,27 @@ func docs(args []string, stdout, stderr io.Writer) int { // a stack CI applies without pinning the provider that answered, and a // stack CI applies with nothing, declared nowhere. Neither is repaired by // regenerating, which is why they belong here rather than in the render. - problems = append(problems, stackProofProblems(*workflow, conformanceRoot, stacksRoot, stacksScript)...) + problems = append(problems, stackProofProblems(*workflow, conformanceRoot, exampleSources())...) // And the class #406 measured: a percentage about an evidence axis, typed // into a page, that nothing ever compared to coverage/evidence.json. Three // of them were wrong when somebody finally did, one by a factor of six. // Regenerating repairs none of it — the sentence has to go — which is why it // belongs here and not among the splices. See docs_axis_figures.go. problems = append(problems, axisFigureProblems(filepath.Dir(*target))...) + // And the class #592 measured: a sentence claiming a client for a pack the + // doorstep refuses. Every row of capabilityMatrix is resolved against the + // instrument it names, and every generated block of the front pages is read + // back for a claim no row carries. Regenerating repairs neither — a row + // whose proof has gone is a decision, and a page claiming a refused pair + // has to stop claiming it. + problems = append(problems, capabilityProblems(*workflow)...) + // The half of it that reads the pages back runs here in --check mode, where + // the documents on disk are the artefact under judgement, and after the + // regeneration in write mode — see capabilityClaimProblems for why the + // order is not a detail. + if *check { + problems = append(problems, capabilityClaimProblems(*workflow, capabilityClaimPages(filepath.Dir(*target)))...) + } if len(problems) > 0 { for _, line := range problems { fmt.Fprintf(stderr, "feint: %s\n", line) @@ -399,6 +431,14 @@ func docs(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "feint: %s no longer matches the versions the suites pin; run `mise run docs:coverage`\n", *clientsPage) return exitDrift } + if promiseChanged { + fmt.Fprintf(stderr, "feint: the promise in the READMEs is behind the capability matrix; run `feint docs`\n") + return exitDrift + } + if capabilityChanged { + fmt.Fprintf(stderr, "feint: the capability matrix published in %s is behind the declared one; run `feint docs`\n", *clientsPage) + return exitDrift + } // The screenshots of the page, on the same gate as every other generated // document. A picture of a screen goes stale exactly like a table of // numbers, and the only difference is that nobody notices for longer. @@ -411,6 +451,24 @@ func docs(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stdout, "%s is up to date\n", *target) return exitOK } + // The target is written here, before the helpers below that also write it. + // Every one of them re-reads the file and splices into what it finds, so a + // target written last would splice `updated` — a copy taken before any of + // them ran — over their work and revert it. + // + // Measured while adding the promise block (#592): a single run that changed + // the coverage tables and the quick start wrote the quick start and then + // put the old one back, and the only symptom was `docs --check` still red + // after a regeneration that reported success. Nothing had lied; the last + // writer won. TestARunThatChangesTwoSectionsOfTheREADMEKeepsBoth fails when + // this write moves back down. + targetChanged := updated != string(current) + if targetChanged { + if err := os.WriteFile(*target, []byte(updated), 0o644); err != nil { //nolint:gosec // a README is world-readable by design + fmt.Fprintf(stderr, "feint: %v\n", err) + return exitError + } + } if safetyChanged { if err := writeSplicedSafety(filepath.Dir(*target)); err != nil { fmt.Fprintf(stderr, "feint: %v\n", err) @@ -499,21 +557,44 @@ func docs(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stdout, "%s updated\n", *installDoc) } if provedChanged { - if err := writeSplicedProved(*clientsPage, *workflow, conformanceRoot, stacksRoot, stacksScript); err != nil { + if err := writeSplicedProved(*clientsPage, *workflow, conformanceRoot, exampleSources()); err != nil { fmt.Fprintf(stderr, "feint: %v\n", err) return exitError } fmt.Fprintf(stdout, "%s updated\n", *clientsPage) } + // Second writer of the clients page, so it re-reads what the one above + // wrote rather than splicing into a copy that predates it. + if capabilityChanged { + if err := writeSplicedCapability(*clientsPage); err != nil { + fmt.Fprintf(stderr, "feint: %v\n", err) + return exitError + } + fmt.Fprintf(stdout, "the capability matrix in %s updated\n", *clientsPage) + } + if promiseChanged { + if err := writeSplicedPromise(filepath.Dir(*target)); err != nil { + fmt.Fprintf(stderr, "feint: %v\n", err) + return exitError + } + fmt.Fprintf(stdout, "the promise in the READMEs now matches the capability matrix\n") + } - if updated == string(current) { + // Read back what was just written. A regeneration repairs a stale block; it + // cannot repair a block whose content claims a client the doorstep refuses, + // and saying so afterwards is the only honest moment to say it. + if claims := capabilityClaimProblems(*workflow, capabilityClaimPages(filepath.Dir(*target))); len(claims) > 0 { + for _, line := range claims { + fmt.Fprintf(stderr, "feint: %s\n", line) + } + fmt.Fprintln(stderr, "feint: the pages were regenerated and still claim it; fix the claim itself") + return exitDrift + } + + if !targetChanged { fmt.Fprintf(stdout, "%s already up to date\n", *target) return exitOK } - if err := os.WriteFile(*target, []byte(updated), 0o644); err != nil { //nolint:gosec // a README is world-readable by design - fmt.Fprintf(stderr, "feint: %v\n", err) - return exitError - } fmt.Fprintf(stdout, "%s updated\n", *target) return exitOK } diff --git a/internal/cli/docs_banner.go b/internal/cli/docs_banner.go index 3d57c3ea..72441c42 100644 --- a/internal/cli/docs_banner.go +++ b/internal/cli/docs_banner.go @@ -216,16 +216,22 @@ func renderSafety(facts safetyFacts, french bool) string { return b.String() } -// bannerPaths answers the documents carrying the banner and whether each is -// French, sorted so two runs report the same order. -func safetyPaths(root string) []struct { +// frontPage is one of the two READMEs, and whether it is the translated one. +// +// One declaration for the three blocks that are now written per locale — the +// safety banner, the promise (#592) and the quick start (#591's fourth finding) +// — because "which pages are the front door, and which of them is French" is +// one fact, and this repository has paid for expressing one fact twice often +// enough. +type frontPage struct { Path string French bool -} { - out := []struct { - Path string - French bool - }{ +} + +// frontPages answers the documents carrying the per-locale blocks, sorted so +// two runs report the same order. +func frontPages(root string) []frontPage { + out := []frontPage{ {filepath.Join(root, "README.md"), false}, {filepath.Join(root, "README.fr.md"), true}, } @@ -244,7 +250,7 @@ func spliceSafety(root string) (bool, error) { if err != nil { return false, err } - for _, doc := range safetyPaths(root) { + for _, doc := range frontPages(root) { current, err := os.ReadFile(doc.Path) //nolint:gosec // a path this repository owns if os.IsNotExist(err) { continue @@ -278,7 +284,7 @@ func writeSplicedSafety(root string) error { if err != nil { return err } - for _, doc := range safetyPaths(root) { + for _, doc := range frontPages(root) { current, err := os.ReadFile(doc.Path) //nolint:gosec // a path this repository owns if os.IsNotExist(err) { continue @@ -310,7 +316,7 @@ func writeSplicedSafety(root string) error { // found. func safetyCarriers(root string) ([]string, error) { var carriers []string - for _, doc := range safetyPaths(root) { + for _, doc := range frontPages(root) { current, err := os.ReadFile(doc.Path) //nolint:gosec // a path this repository owns if os.IsNotExist(err) { continue diff --git a/internal/cli/docs_clients.go b/internal/cli/docs_clients.go index ebaec948..2802e97a 100644 --- a/internal/cli/docs_clients.go +++ b/internal/cli/docs_clients.go @@ -52,15 +52,23 @@ const ( // It deliberately no longer says which provider the client proves: that is the // column this file exists to derive. A client the workflow drives and this list // does not name is an error in renderClients, not a silent omission. +// It gained two columns with #592, and they belong here rather than beside the +// capability matrix for the reason this repository states about every fact +// expressed twice: `token` is the word prose uses for the client, and `engine` +// is the `iac.engine` name when the client is one `feint up` runs. A second +// list of client names next door would be a second thing to keep in step, and +// capability.go asks this one instead. var clientSources = []struct { name string variable string + token string + engine string }{ - {"`scw`", "SCW_VERSION"}, - {"Terraform", "TERRAFORM_VERSION"}, - {"OpenTofu", "TOFU_VERSION"}, - {"`octl`", "OCTL_VERSION"}, - {"`exo`", "EXO_VERSION"}, + {"`scw`", "SCW_VERSION", "scw", ""}, + {"Terraform", "TERRAFORM_VERSION", "terraform", "terraform"}, + {"OpenTofu", "TOFU_VERSION", "opentofu", "opentofu"}, + {"`octl`", "OCTL_VERSION", "octl", ""}, + {"`exo`", "EXO_VERSION", "exo", ""}, } var ( diff --git a/internal/cli/docs_proved.go b/internal/cli/docs_proved.go index a067f891..76a456c8 100644 --- a/internal/cli/docs_proved.go +++ b/internal/cli/docs_proved.go @@ -59,8 +59,63 @@ const ( // is what decides which of them CI actually applies. stacksRoot = "examples/stacks" stacksScript = "tools/conformance/stacks.sh" + + // quickstartRoot holds the other kind of example, and the split is #593: + // examples/stacks/scaleway is 625 lines of two VPCs, ACLs, a bastion, a + // balancer and a golden image. It is an excellent qualification stack — it + // found #249 and #250, which every other gate was blind to — and a poor + // thing to read first, and it was being asked to be both. So the first + // success gets a directory of its own: a provider, an address, a server, + // short enough to read whole. + // + // It is held to the same terms as the qualification stacks rather than + // weaker ones, because "an example nobody runs is an example that rots" is + // the sentence stacks.sh opens with and a quickstart is the example most + // people run. Same population, same two refusals: applied by CI or declared + // with a reason, and pinning the provider that answered. + quickstartRoot = "examples/quickstart" + quickstartScript = "tools/conformance/quickstart.sh" + // quickstartLead is the pack the README's first door lands in. One name, + // read by the generator and by nothing else, so the page cannot send a + // reader to a directory that moved. + quickstartLead = "scaleway" ) +// exampleSource is one directory of example configurations and the script that +// applies them. +// +// Two of them since #593, and the plural is the point: every refusal in +// docs_stacks.go walks this list, so the quickstart examples are judged by the +// checks the qualification stacks are judged by rather than living one +// directory away from them. Adding a root here is what wires a new family in; +// forgetting to is what the second direction of undeclaredStacks catches. +type exampleSource struct { + // Family is the repository-relative root, and it is the source's identity: + // what a declaration in stacksRunByHand names, what a refusal prints, and + // what keys the "is this applied" answer. Separate from Root for the reason + // CIRef is separate from Script — a test walks a copy, and a declaration + // about `examples/stacks/exoscale` must still be the declaration about it. + Family string + // Root is the directory to walk. A test may point it at a copy. + Root string + // Script is the suite whose `run_stack` lines are read. A test may point it + // at a copy too. + Script string + // CIRef is the repository-relative path the workflow has to name for the + // script to count as run in CI. Separate from Script for the reason + // stacksAppliedInCI already documented: a test renders from a copy + // elsewhere on disk, and looking the copy's own path up in the workflow + // would answer "CI does not run it" about every source. + CIRef string +} + +func exampleSources() []exampleSource { + return []exampleSource{ + {Family: stacksRoot, Root: stacksRoot, Script: stacksScript, CIRef: stacksScript}, + {Family: quickstartRoot, Root: quickstartRoot, Script: quickstartScript, CIRef: quickstartScript}, + } +} + // pinnedIn names the file every client version comes from. One string, so the // page cannot credit a pin to a file that does not hold it. const pinnedIn = conformanceWorkflow @@ -240,25 +295,33 @@ func providerPins(root string, driven func(dir string) bool) ([]providerPin, err // Both halves are read: the script that names them, and the workflow that runs // the script. A stack CI never applies proves nothing, whatever it pins, and // examples/stacks/exoscale is exactly that today. -func stacksAppliedInCI(script, workflow string) (map[string]bool, error) { - body, err := os.ReadFile(script) //nolint:gosec // a path this repository owns - if err != nil { - return nil, err - } +// The answer is keyed by directory, `examples/stacks/scaleway`, rather than by +// bare name: two roots since #593, and `scaleway` names a stack under each of +// them. A set of names would have credited the quickstart with the +// qualification stack's run, which is the understated-proof half of the defect +// the client matrix was filed for. +func stacksAppliedInCI(sources []exampleSource, workflow string) (map[string]bool, error) { flow, err := os.ReadFile(workflow) //nolint:gosec // a path this repository owns if err != nil { return nil, err } out := map[string]bool{} - // The workflow is searched for the repository-relative path a workflow can - // actually carry, not for the path this function was handed: a test renders - // from a copy elsewhere on disk, and comparing against that copy's own path - // would answer "CI does not run it" about every one of them. - if !strings.Contains(string(flow), stacksScript) { - return out, nil - } - for _, m := range runStack.FindAllStringSubmatch(string(body), -1) { - out[m[1]] = true + for _, source := range sources { + body, err := os.ReadFile(source.Script) //nolint:gosec // a path this repository owns + if err != nil { + return nil, err + } + // The workflow is searched for the repository-relative path a workflow + // can actually carry, not for the path this function was handed: a test + // renders from a copy elsewhere on disk, and comparing against that + // copy's own path would answer "CI does not run it" about every one of + // them. + if !strings.Contains(string(flow), source.CIRef) { + continue + } + for _, m := range runStack.FindAllStringSubmatch(string(body), -1) { + out[source.Family+"/"+m[1]] = true + } } return out, nil } @@ -382,7 +445,7 @@ func unreadablePins(workflow string) ([]string, error) { } // renderProved builds the page. -func renderProved(workflow, root, stacks, script string) (string, error) { +func renderProved(workflow, root string, sources []exampleSource) (string, error) { versions, err := pinnedVersions(workflow) if err != nil { return "", err @@ -440,7 +503,7 @@ func renderProved(workflow, root, stacks, script string) (string, error) { "nothing installs", workflow, strings.Join(unused, ", ")) } - pins, err := providerPinsOfRepository(workflow, root, stacks, script) + pins, err := providerPinsOfRepository(workflow, root, sources) if err != nil { return "", err } @@ -455,8 +518,8 @@ func renderProved(workflow, root, stacks, script string) (string, error) { // refusal in docs_stacks.go that will not let a directory be applied in CI // without saying which provider versions it accepts. Computing it twice would // be two chances for the page and the refusal to disagree about the same row. -func providerPinsOfRepository(workflow, root, stacks, script string) ([]providerPin, error) { - appliedStacks, err := stacksAppliedInCI(script, workflow) +func providerPinsOfRepository(workflow, root string, sources []exampleSource) ([]providerPin, error) { + appliedStacks, err := stacksAppliedInCI(sources, workflow) if err != nil { return nil, err } @@ -468,29 +531,37 @@ func providerPinsOfRepository(workflow, root, stacks, script string) ([]provider if appliedFixtures[dir] { return true } - // The first segment under the stacks root, not the whole tail: a stack - // is applied with its `modules/` subtree copied beside it, so + // The first segment under the source's root, not the whole tail: a + // stack is applied with its `modules/` subtree copied beside it, so // examples/stacks/outscale/modules/net is applied exactly when // examples/stacks/outscale is. Comparing the whole tail said no, which // would have understated a proof — the failure the client matrix was // filed for, in its other direction. - rest := strings.TrimPrefix(dir, stacks+"/") - if rest == dir { - return false + for _, source := range sources { + rest := strings.TrimPrefix(dir, source.Root+"/") + if rest == dir { + continue + } + name, _, _ := strings.Cut(rest, "/") + if appliedStacks[source.Family+"/"+name] { + return true + } } - name, _, _ := strings.Cut(rest, "/") - return appliedStacks[name] + return false } - fixturePins, err := providerPins(root, driven) + pins, err := providerPins(root, driven) if err != nil { return nil, err } - stackPins, err := providerPins(stacks, driven) - if err != nil { - return nil, err + for _, source := range sources { + more, err := providerPins(source.Root, driven) + if err != nil { + return nil, err + } + pins = append(pins, more...) } - return append(fixturePins, stackPins...), nil + return pins, nil } // renderProved builds the page, continued. @@ -550,7 +621,7 @@ func renderProvedTables(versions map[string]string, proofs map[string][]clientPr } // spliceProved reports whether the page is out of date, and leaves it alone. -func spliceProved(path, workflow, root, stacks, script string) (bool, error) { +func spliceProved(path, workflow, root string, sources []exampleSource) (bool, error) { current, err := os.ReadFile(path) //nolint:gosec // a path this repository owns if os.IsNotExist(err) { return false, nil @@ -561,7 +632,7 @@ func spliceProved(path, workflow, root, stacks, script string) (bool, error) { if !strings.Contains(string(current), provedStartMarker) { return false, nil } - rendered, err := renderProved(workflow, root, stacks, script) + rendered, err := renderProved(workflow, root, sources) if err != nil { return false, err } @@ -572,12 +643,12 @@ func spliceProved(path, workflow, root, stacks, script string) (bool, error) { return updated != string(current), nil } -func writeSplicedProved(path, workflow, root, stacks, script string) error { +func writeSplicedProved(path, workflow, root string, sources []exampleSource) error { current, err := os.ReadFile(path) //nolint:gosec // same path as above if err != nil { return err } - rendered, err := renderProved(workflow, root, stacks, script) + rendered, err := renderProved(workflow, root, sources) if err != nil { return err } diff --git a/internal/cli/docs_proved_test.go b/internal/cli/docs_proved_test.go index ce7a08c7..fa1a2d77 100644 --- a/internal/cli/docs_proved_test.go +++ b/internal/cli/docs_proved_test.go @@ -14,8 +14,7 @@ func provedPage(t *testing.T) string { rendered, err := renderProved( filepath.Join(root, conformanceWorkflow), filepath.Join(root, conformanceRoot), - filepath.Join(root, stacksRoot), - filepath.Join(root, stacksScript), + repoSources(t), ) if err != nil { t.Fatalf("render the proved-against page: %v", err) @@ -23,6 +22,29 @@ func provedPage(t *testing.T) string { return rendered } +// repoSources is the example roots as they stand, with absolute paths. +func repoSources(t *testing.T) []exampleSource { + t.Helper() + _, _, sources := stackPaths(t) + return sources +} + +// sourcesWithScript is the same list with one family's suite pointed at a copy, +// which is how a test asks "what would the page say if CI stopped applying +// this" without editing the repository. +func sourcesWithScript(t *testing.T, ciRef, script string) []exampleSource { + t.Helper() + sources := repoSources(t) + for i := range sources { + if sources[i].CIRef == ciRef { + sources[i].Script = script + return sources + } + } + t.Fatalf("no example source runs %s", ciRef) + return nil +} + // rowContaining returns the single table row naming key, and fails when there // is none. // @@ -110,8 +132,7 @@ func TestEveryPinnedVersionIsInstalledByAStepThatUsesIt(t *testing.T) { if err := os.WriteFile(copied, []byte(hardCoded), 0o600); err != nil { t.Fatal(err) } - _, err = renderProved(copied, filepath.Join(root, conformanceRoot), - filepath.Join(root, stacksRoot), filepath.Join(root, stacksScript)) + _, err = renderProved(copied, filepath.Join(root, conformanceRoot), repoSources(t)) if err == nil { t.Fatal("the page renders a pinned version no step reads, so it would publish a number " + "nothing installs") @@ -142,8 +163,7 @@ func TestAPinnedVersionNoClientClaimsIsRefused(t *testing.T) { t.Fatal(err) } - _, err = renderProved(copied, filepath.Join(root, conformanceRoot), - filepath.Join(root, stacksRoot), filepath.Join(root, stacksScript)) + _, err = renderProved(copied, filepath.Join(root, conformanceRoot), repoSources(t)) if err == nil { t.Fatal("a version CI pins is silently left off the page") } @@ -188,8 +208,7 @@ func TestAPinDeclaredInAFormTheReaderCannotSeeIsRefused(t *testing.T) { t.Fatal(err) } - _, err = renderProved(copied, filepath.Join(root, conformanceRoot), - filepath.Join(root, stacksRoot), filepath.Join(root, stacksScript)) + _, err = renderProved(copied, filepath.Join(root, conformanceRoot), repoSources(t)) if err == nil { t.Fatal("a pin the reader cannot parse renders as \"not pinned\" instead of being refused") } @@ -232,7 +251,7 @@ func TestAFixtureNoSuiteAppliesIsNotAProof(t *testing.T) { t.Fatal(err) } without, err := renderProved(filepath.Join(root, conformanceWorkflow), - filepath.Join(root, conformanceRoot), filepath.Join(root, stacksRoot), copied) + filepath.Join(root, conformanceRoot), sourcesWithScript(t, stacksScript, copied)) if err != nil { t.Fatalf("render from the trimmed script: %v", err) } @@ -264,8 +283,7 @@ func TestAFixtureNoSuiteAppliesIsNotAProof(t *testing.T) { if err := os.WriteFile(copiedFlow, []byte(unrun), 0o600); err != nil { t.Fatal(err) } - page, err := renderProved(copiedFlow, filepath.Join(root, conformanceRoot), - filepath.Join(root, stacksRoot), filepath.Join(root, stacksScript)) + page, err := renderProved(copiedFlow, filepath.Join(root, conformanceRoot), repoSources(t)) if err != nil { t.Fatalf("render from the trimmed workflow: %v", err) } diff --git a/internal/cli/docs_quickstart.go b/internal/cli/docs_quickstart.go index 742245c0..13db174c 100644 --- a/internal/cli/docs_quickstart.go +++ b/internal/cli/docs_quickstart.go @@ -15,11 +15,22 @@ import ( // needs in the first ten seconds is *my Terraform ran and no cloud account was // involved*, and it was four screens down. // -// Two doors rather than one, and that is the other half. The OCI image is the -// format a CI actually consumes — a `services:` block, a compose file, a -// Testcontainers call — and it appeared nowhere in the README at all while -// living in docs/install.md. A reader deciding whether Feint fits their pipeline -// was being shown the mode that fits a laptop. +// **And then it taught the version before the one this project ships (#593).** +// 0.11 introduced `feint.yaml`, `feint up` and `feint down`; this block still +// opened with `feint start` / `eval "$(feint env scaleway)"` / `terraform +// apply`. Worse than old: **not copyable**. In which directory, with which +// `main.tf`, with which provider block? The `Apply complete!` printed under +// those three lines was not reachable from them, which is the one thing a quick +// start may never be. So the first door is now four commands against +// examples/quickstart/scaleway — a stack short enough to read whole — and the +// three-line hand-driven form stays underneath, where it is what a reader with +// an existing project actually needs. +// +// Two more doors, and that is the other half. The OCI image is the format a CI +// actually consumes — a `services:` block, a compose file, a Testcontainers +// call — and it appeared nowhere in the README at all while living in +// docs/install.md. A reader deciding whether Feint fits their pipeline was +// being shown the mode that fits a laptop. // // The third door is the Marketplace action (#245): `uses: // stephrobert/setup-feint@v1` is the line people copy out of somebody else's @@ -32,14 +43,27 @@ import ( // is: the image tag is a release's own, and a hand-written one is wrong the day // after the next release. So is the `version:` under the action. `feint docs // --check` fails until it is regenerated. +// +// **Rendered per locale since #591.** The block used to be injected in English +// into both READMEs, on the rule README.fr.md states about every generated +// block: *a command needs no translation*. True of a command, and this block is +// not only commands — it carried "On your machine" and "In CI, or anywhere +// Docker runs" into the middle of a French page. What stays shared is what is +// structured: the version, the image, the repository, the commands themselves. +// TestTheFrenchQuickStartIsInFrench fails when an English sentence comes back. const ( quickstartStartMarker = "" quickstartEndMarker = "" ) -// renderQuickstart writes the two doors, both pinned to the released version. -func renderQuickstart(goMod, changelog string) (string, error) { +// renderQuickstart writes the doors, all of them pinned to the released +// version, in the locale of the page that carries them. +// +// nolint:misspell // the French half is French: "ressemble" is a word, and the +// linter reads every Go string as English. The same exemption renderSafety +// carries, for the same reason. +func renderQuickstart(goMod, changelog string, french bool) (string, error) { slug := repositorySlug(goMod) if slug == "" { return "", fmt.Errorf("cannot render the quick start: no module path in %s", goMod) @@ -49,63 +73,202 @@ func renderQuickstart(goMod, changelog string) (string, error) { return "", fmt.Errorf("cannot render the quick start: no released section in %s to pin it to", changelog) } image := "ghcr.io/" + strings.ToLower(slug) + // The stack the first door lands in. Named from the same constant the gate + // that applies it reads, so the README cannot send a reader to a directory + // no suite keeps working. + stack := quickstartRoot + "/" + quickstartLead var b strings.Builder b.WriteString(docsGenerated) b.WriteString("\n\n") - b.WriteString("**On your machine** — one static binary, nothing else:\n\n") + // The French half of this block avoids the em-dash throughout, which is a + // style rule for this repository's French prose and not a rendering + // accident: a colon, a comma pair or a full stop carries every one of the + // breaks the English half sets with `—`. + if french { + b.WriteString("**Sur votre machine**, un binaire statique et une stack assez courte pour se lire :\n\n") + } else { + b.WriteString("**On your machine** — one static binary, and a stack short enough to read whole:\n\n") + } b.WriteString("```bash\n") - b.WriteString("feint start # detaches, waits until it answers\n") - b.WriteString("eval \"$(feint env scaleway)\" # point the official client at it\n") - b.WriteString("terraform apply # the real Scaleway provider\n") + fmt.Fprintf(&b, "brew install %s/%s\n", slug, pathBase(slug)) + fmt.Fprintf(&b, "git clone https://github.com/%s\n", slug) + fmt.Fprintf(&b, "cd %s/%s\n", pathBase(slug), stack) + if french { + b.WriteString("feint up # vérifie la station, démarre l'émulateur, applique le Terraform\n") + b.WriteString("feint down # détruit ce qu'il a créé, puis arrête l'émulateur\n") + } else { + b.WriteString("feint up # checks the host, starts the emulator, applies the Terraform\n") + b.WriteString("feint down # destroys what it created, then stops the emulator\n") + } b.WriteString("```\n\n") - b.WriteString("**In CI, or anywhere Docker runs** — the same emulator as a service:\n\n") + // What those four commands print, derived from the configuration they run + // rather than typed beside it. + // + // The line that used to sit under this block said `Resources: 5 added` and + // was #593's second finding: it was not reachable from the commands above + // it — three lines with no directory, no `main.tf` and no provider block — + // so a reader following the quick start exactly could not arrive at the + // output the quick start displayed. It is derived now, and + // tools/conformance/quickstart.sh takes this very line out of the README and + // requires the run to print it, so the output shown is the output produced. + // Anchored on the module file the version already comes from, so the three + // facts this block carries are read relative to one root rather than to + // whatever directory the binary was run in. + added, err := resourceCount(filepath.Join(filepath.Dir(goMod), quickstartRoot, quickstartLead, "main.tf")) + if err != nil { + return "", err + } + if french { + b.WriteString("Ce que ces quatre commandes impriment :\n\n") + } else { + b.WriteString("What those four commands print:\n\n") + } + b.WriteString("```text\n") + fmt.Fprintf(&b, "Apply complete! Resources: %d added, 0 changed, 0 destroyed.\n", added) + b.WriteString("```\n\n") + + if french { + fmt.Fprintf(&b, "`feint up` lit le `feint.yaml` posé à côté du Terraform : il vérifie la station, "+ + "démarre l'émulateur, exporte ce dont le client officiel a besoin, lance le moteur et "+ + "attend les conditions que le fichier déclare. La stack fait un serveur et une adresse, "+ + "sans runtime de machines. `%s/%s` est celle qui ressemble à de la production, et elle "+ + "est faite pour casser l'émulateur plutôt que pour se lire en premier.\n\n", + stacksRoot, quickstartLead) + b.WriteString("**Vous avez déjà un projet ?** Pilotez-le à la main, c'est ce qu'un `feint.yaml` " + + "vous évite d'écrire :\n\n") + } else { + fmt.Fprintf(&b, "`feint up` reads the `feint.yaml` beside the Terraform: it checks the host, "+ + "starts the emulator, exports what the official client needs, runs the engine and waits "+ + "on the conditions the file declares. The stack is one server and one address, with no "+ + "machine runtime — `%s/%s` is the one shaped like production, and it exists to break the "+ + "emulator rather than to be read first.\n\n", + stacksRoot, quickstartLead) + b.WriteString("**Already have a project?** Drive it by hand, which is what a `feint.yaml` " + + "saves you from writing:\n\n") + } + b.WriteString("```bash\n") + if french { + b.WriteString("feint start # se détache, attend qu'il réponde\n") + b.WriteString("eval \"$(feint env scaleway)\" # pointe le client officiel vers lui\n") + b.WriteString("terraform apply # le vrai provider Scaleway\n") + } else { + b.WriteString("feint start # detaches, waits until it answers\n") + b.WriteString("eval \"$(feint env scaleway)\" # point the official client at it\n") + b.WriteString("terraform apply # the real Scaleway provider\n") + } + b.WriteString("```\n\n") + + if french { + b.WriteString("**En CI, ou partout où Docker tourne**, le même émulateur en service :\n\n") + } else { + b.WriteString("**In CI, or anywhere Docker runs** — the same emulator as a service:\n\n") + } b.WriteString("```yaml\n") b.WriteString("services:\n") b.WriteString(" feint:\n") fmt.Fprintf(&b, " image: %s:v%s\n", image, version) b.WriteString(" ports: [\"4599:4599\"]\n") b.WriteString("```\n\n") - fmt.Fprintf(&b, "The image is control-plane only and carries no `latest` tag; "+ - "[docs/install.md](docs/install.md) has the GitLab form, the compose form and "+ - "the signature verification. Pull it directly with "+ - "`docker run --rm -p 127.0.0.1:4599:4599 %s:v%s`.\n\n", image, version) - - b.WriteString("**In GitHub Actions without a container** — the action from the Marketplace:\n\n") + if french { + fmt.Fprintf(&b, "L'image ne sert que le plan de contrôle et ne porte aucun tag `latest` ; "+ + "[docs/install.md](docs/install.md) donne la forme GitLab, la forme compose et la "+ + "vérification de signature. Pour la tirer directement : "+ + "`docker run --rm -p 127.0.0.1:4599:4599 %s:v%s`.\n\n", image, version) + b.WriteString("**Dans GitHub Actions, sans conteneur**, l'action du Marketplace :\n\n") + } else { + fmt.Fprintf(&b, "The image is control-plane only and carries no `latest` tag; "+ + "[docs/install.md](docs/install.md) has the GitLab form, the compose form and "+ + "the signature verification. Pull it directly with "+ + "`docker run --rm -p 127.0.0.1:4599:4599 %s:v%s`.\n\n", image, version) + b.WriteString("**In GitHub Actions without a container** — the action from the Marketplace:\n\n") + } b.WriteString("```yaml\n") b.WriteString("- uses: stephrobert/setup-feint@v1\n") b.WriteString(" with:\n") fmt.Fprintf(&b, " version: %s\n", version) - b.WriteString(" provider: scaleway # exports what the official client needs\n") + if french { + b.WriteString(" provider: scaleway # exporte ce dont le client officiel a besoin\n") + } else { + b.WriteString(" provider: scaleway # exports what the official client needs\n") + } b.WriteString("```\n\n") - b.WriteString("It installs the released binary, **verifies its checksum before running " + - "it**, and waits until the emulator answers.\n") + if french { + b.WriteString("Elle installe le binaire publié, **vérifie sa somme de contrôle avant de " + + "l'exécuter**, et attend que l'émulateur réponde.\n") + } else { + b.WriteString("It installs the released binary, **verifies its checksum before running " + + "it**, and waits until the emulator answers.\n") + } return b.String(), nil } -// Both READMEs carry it, and the block itself stays in English in both — the -// rule README.fr.md states about every generated block, for the reason it gives: -// they are rendered from the code rather than written by hand. The prose around -// them is translated; a command is not. +// resourceCount answers how many objects an apply of this configuration +// creates: one per `resource` block. // -// The English one alone would have been the mistake this repository already made -// once: the French command table had fallen ten verbs behind and nobody was -// looking, because no check read that page (#237). -func quickstartPaths(root string) []string { - return []string{ - filepath.Join(root, "README.md"), - filepath.Join(root, "README.fr.md"), +// That equality is only true while no block multiplies itself, so a `count` or +// a `for_each` is refused rather than counted wrong. Refusing is the right +// answer here and not a limitation of the parser: the number goes on the front +// page as what a reader will see, and a quick start short enough to read whole +// has no business multiplying resources. A quickstart that needs one has +// outgrown the job, and the refusal says so instead of publishing a figure that +// is quietly off by two. +func resourceCount(path string) (int, error) { + body, err := os.ReadFile(path) //nolint:gosec // a path this repository owns + if err != nil { + return 0, fmt.Errorf("cannot count what the quick start creates: %w", err) } + count := 0 + for _, line := range strings.Split(string(body), "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "#") { + continue + } + if strings.HasPrefix(trimmed, "resource \"") { + count++ + continue + } + if strings.HasPrefix(trimmed, "count ") || strings.HasPrefix(trimmed, "count=") || + strings.HasPrefix(trimmed, "for_each ") || strings.HasPrefix(trimmed, "for_each=") { + return 0, fmt.Errorf( + "%s uses %q: one resource block is no longer one object, so the number of resources "+ + "the quick start prints cannot be derived from the file. Either drop it, or this "+ + "example has outgrown being the first one somebody reads", + path, strings.Fields(trimmed)[0]) + } + } + if count == 0 { + return 0, fmt.Errorf("%s declares no resource: the quick start would tell a reader to run "+ + "an apply that creates nothing", path) + } + return count, nil +} + +// pathBase is the repository's own directory name, the one `git clone` creates. +func pathBase(slug string) string { + if i := strings.LastIndex(slug, "/"); i >= 0 { + return slug[i+1:] + } + return slug } +// Both READMEs carry it, each in its own language since #591. Sharing the +// English block was the rule README.fr.md still states about generated blocks — +// *a command needs no translation* — applied one word too far: this one also +// carries sentences, and they were landing in English in the middle of a French +// page. What is shared is the structure the code owns. +// +// Rendering only the English one would be the mistake this repository already +// made once: the French command table had fallen ten verbs behind and nobody was +// looking, because no check read that page (#237). func spliceQuickstart(root, goMod, changelog string) (bool, error) { if root == "" { return false, nil } - for _, path := range quickstartPaths(root) { - current, err := os.ReadFile(path) //nolint:gosec // a path this repository owns + for _, page := range frontPages(root) { + current, err := os.ReadFile(page.Path) //nolint:gosec // a path this repository owns if err != nil { if os.IsNotExist(err) { continue @@ -115,13 +278,13 @@ func spliceQuickstart(root, goMod, changelog string) (bool, error) { if !strings.Contains(string(current), quickstartStartMarker) { continue } - rendered, err := renderQuickstart(goMod, changelog) + rendered, err := renderQuickstart(goMod, changelog, page.French) if err != nil { return false, err } updated, err := spliceSection(string(current), quickstartStartMarker, quickstartEndMarker, rendered) if err != nil { - return false, fmt.Errorf("%s: %w", path, err) + return false, fmt.Errorf("%s: %w", page.Path, err) } if updated != string(current) { return true, nil @@ -131,8 +294,8 @@ func spliceQuickstart(root, goMod, changelog string) (bool, error) { } func writeSplicedQuickstart(root, goMod, changelog string) error { - for _, path := range quickstartPaths(root) { - current, err := os.ReadFile(path) //nolint:gosec // a path this repository owns + for _, page := range frontPages(root) { + current, err := os.ReadFile(page.Path) //nolint:gosec // a path this repository owns if err != nil { if os.IsNotExist(err) { continue @@ -142,15 +305,15 @@ func writeSplicedQuickstart(root, goMod, changelog string) error { if !strings.Contains(string(current), quickstartStartMarker) { continue } - rendered, err := renderQuickstart(goMod, changelog) + rendered, err := renderQuickstart(goMod, changelog, page.French) if err != nil { return err } updated, err := spliceSection(string(current), quickstartStartMarker, quickstartEndMarker, rendered) if err != nil { - return fmt.Errorf("%s: %w", path, err) + return fmt.Errorf("%s: %w", page.Path, err) } - if err := os.WriteFile(path, []byte(updated), 0o644); err != nil { //nolint:gosec // documentation is world-readable by design + if err := os.WriteFile(page.Path, []byte(updated), 0o644); err != nil { //nolint:gosec // documentation is world-readable by design return err } } diff --git a/internal/cli/docs_stacks.go b/internal/cli/docs_stacks.go index 831e5f7a..2de508ad 100644 --- a/internal/cli/docs_stacks.go +++ b/internal/cli/docs_stacks.go @@ -51,7 +51,12 @@ import ( // stackException is one example stack the conformance suite does not apply, and // why. type stackException struct { - // Stack is the directory name directly under examples/stacks. + // Root is the example family the stack belongs to, one of the roots + // exampleSources names. Written out since #593 put a second family beside + // the first: `scaleway` names a directory under each of them, and a + // declaration keyed by name alone would have excused both. + Root string + // Stack is the directory name directly under Root. Stack string // Reason says what stops CI applying it, and what is done instead. Empty is // refused: "not run" and "not run because X, applied by hand on D" are @@ -66,6 +71,7 @@ type stackException struct { // would be inconvenient. var stacksRunByHand = []stackException{ { + Root: stacksRoot, Stack: "exoscale", Reason: "suspended — no Terraform for Exoscale until upstream " + "exoscale/terraform-provider-exoscale#573 is fixed: the published provider honours " + @@ -108,15 +114,21 @@ func stackDirs(root string) ([]string, error) { // because `feint docs` also regenerates a README outside it — see the note at // the top of this file, and TestTheStackChecksHaveASubjectToMeasure, which is // what keeps that from becoming a check nobody runs. -func stackProofProblems(workflow, root, stacks, script string) []string { - if _, err := os.Stat(stacks); os.IsNotExist(err) { +func stackProofProblems(workflow, root string, sources []exampleSource) []string { + present := sources[:0:0] + for _, source := range sources { + if _, err := os.Stat(source.Root); err == nil { + present = append(present, source) + } + } + if len(present) == 0 { return nil } - problems := undeclaredStacks(stacks, script, workflow) - pins, err := providerPinsOfRepository(workflow, root, stacks, script) + problems := undeclaredStacks(present, workflow) + pins, err := providerPinsOfRepository(workflow, root, present) if err != nil { return append(problems, fmt.Sprintf( - "cannot read the provider constraints under %s and %s: %v", root, stacks, err)) + "cannot read the provider constraints under %s and the example roots: %v", root, err)) } return append(problems, unconstrainedAppliedPins(pins)...) } @@ -130,55 +142,72 @@ func stackProofProblems(workflow, root, stacks, script string) []string { // - a stack CI does not apply and nobody declared; // - a declaration naming a stack that does not exist; // - a declaration for a stack CI does apply, or with no reason given. -func undeclaredStacks(root, script, workflow string) []string { - stacks, err := stackDirs(root) - if err != nil { - return []string{err.Error()} - } - applied, err := stacksAppliedInCI(script, workflow) +func undeclaredStacks(sources []exampleSource, workflow string) []string { + applied, err := stacksAppliedInCI(sources, workflow) if err != nil { return []string{fmt.Sprintf("cannot read which stacks CI applies: %v", err)} } + // Keyed by directory rather than by name, because two roots hold a + // `scaleway`: a declaration excusing one of them must not excuse the other. declared := map[string]string{} + scriptOf := map[string]string{} var problems []string for _, e := range stacksRunByHand { + key := e.Root + "/" + e.Stack if strings.TrimSpace(e.Reason) == "" { problems = append(problems, fmt.Sprintf( - "%s/%s is declared as run by hand with no reason: \"not applied\" and \"not applied "+ - "because X\" are different facts, and only the second is a decision", - root, e.Stack)) + "%s is declared as run by hand with no reason: \"not applied\" and \"not applied "+ + "because X\" are different facts, and only the second is a decision", key)) } - declared[e.Stack] = e.Reason + declared[key] = e.Reason } existing := map[string]bool{} - for _, stack := range stacks { - existing[stack] = true - if applied[stack] { + for _, source := range sources { + scriptOf[source.Family] = source.CIRef + stacks, err := stackDirs(source.Root) + if err != nil { + problems = append(problems, err.Error()) continue } - if _, ok := declared[stack]; !ok { - problems = append(problems, fmt.Sprintf( - "%s/%s is applied by no `run_stack` line in %s and stacksRunByHand in "+ - "internal/cli/docs_stacks.go does not declare why: a stack nobody applies is not a "+ - "proof, and the table would print `no` for it as if that were a decision", - root, stack, script)) + for _, stack := range stacks { + key := source.Family + "/" + stack + existing[key] = true + if applied[key] { + continue + } + if _, ok := declared[key]; !ok { + problems = append(problems, fmt.Sprintf( + "%s is applied by no `run_stack` line in %s and stacksRunByHand in "+ + "internal/cli/docs_stacks.go does not declare why: a stack nobody applies is not a "+ + "proof, and the table would print `no` for it as if that were a decision", + key, source.CIRef)) + } } } for _, e := range stacksRunByHand { - if !existing[e.Stack] { + key := e.Root + "/" + e.Stack + // A declaration naming a root nothing walks excuses nothing either, and + // it is the shape a typo takes. + if _, walked := scriptOf[e.Root]; !walked { problems = append(problems, fmt.Sprintf( - "stacksRunByHand declares %s/%s and no such stack exists: a declaration that "+ - "excuses nothing is a reason nobody re-reads", root, e.Stack)) + "stacksRunByHand declares %s and %s is not one of the example roots: a declaration "+ + "nothing reads is a reason nobody re-reads", key, e.Root)) continue } - if applied[e.Stack] { + if !existing[key] { problems = append(problems, fmt.Sprintf( - "stacksRunByHand says %s/%s is run by hand and %s applies it: the reason is stale, "+ + "stacksRunByHand declares %s and no such stack exists: a declaration that "+ + "excuses nothing is a reason nobody re-reads", key)) + continue + } + if applied[key] { + problems = append(problems, fmt.Sprintf( + "stacksRunByHand says %s is run by hand and %s applies it: the reason is stale, "+ "and it is the kind that survives for months because it reads like evidence", - root, e.Stack, script)) + key, scriptOf[e.Root])) } } sort.Strings(problems) @@ -227,9 +256,14 @@ func renderStackExceptions() string { b.WriteString("merely absent, so a `no` is a decision somebody wrote down and not a stack\n") b.WriteString("nobody wired up:\n\n") ordered := append([]stackException{}, stacksRunByHand...) - sort.Slice(ordered, func(i, j int) bool { return ordered[i].Stack < ordered[j].Stack }) + sort.Slice(ordered, func(i, j int) bool { + if ordered[i].Root != ordered[j].Root { + return ordered[i].Root < ordered[j].Root + } + return ordered[i].Stack < ordered[j].Stack + }) for _, e := range ordered { - b.WriteString(wrapBullet(fmt.Sprintf("`%s/%s` — %s.", stacksRoot, e.Stack, e.Reason))) + b.WriteString(wrapBullet(fmt.Sprintf("`%s/%s` — %s.", e.Root, e.Stack, e.Reason))) } return b.String() } @@ -237,27 +271,49 @@ func renderStackExceptions() string { // wrapBullet renders one list item wrapped the way the prose around it is, so a // declared reason does not land as a single 500-character line in a file every // other paragraph of which stops at the same column. -func wrapBullet(text string) string { +// +// It never breaks inside an inline code span. Markdown renders `feint\nup` as +// code either way, but a reader opening the file meets a command cut in half, +// and every other paragraph in these documents keeps its commands whole. The +// rule is the parity of the backticks seen so far: inside a span, the line runs +// past the column rather than break. +func wrapBullet(text string) string { return wrapAt(text, "- ", " ") } + +// wrapParagraph is the same wrapping for prose that is not a list item, so a +// generated sentence whose length depends on how many packs there are does not +// land as one long line in a file every other paragraph of which stops at the +// same column. A hand-placed newline in a format string cannot do this: the +// names it separates are not a fixed width. +func wrapParagraph(text string) string { return wrapAt(text, "", "") } + +// wrapAt wraps to the column these documents use, never breaking inside an +// inline code span. +func wrapAt(text, first, indent string) string { const width = 76 var b strings.Builder column := 0 - prefix := "- " + prefix := first + inCode := false for _, word := range strings.Fields(text) { switch { case column == 0: b.WriteString(prefix) b.WriteString(word) column = len(prefix) + len(word) - prefix = " " - case column+1+len(word) > width: - b.WriteString("\n ") + prefix = indent + case !inCode && column+1+len(word) > width: + b.WriteString("\n") + b.WriteString(indent) b.WriteString(word) - column = 2 + len(word) + column = len(indent) + len(word) default: b.WriteString(" ") b.WriteString(word) column += 1 + len(word) } + if strings.Count(word, "`")%2 == 1 { + inCode = !inCode + } } b.WriteString("\n") return b.String() diff --git a/internal/cli/docs_stacks_test.go b/internal/cli/docs_stacks_test.go index 8f701899..53e216ae 100644 --- a/internal/cli/docs_stacks_test.go +++ b/internal/cli/docs_stacks_test.go @@ -7,48 +7,89 @@ import ( "testing" ) -// stackPaths gives the four paths the stack checks read, rooted in the -// repository. -func stackPaths(t *testing.T) (workflow, root, stacks, script string) { +// stackPaths gives the paths the stack checks read, rooted in the repository. +// +// The sources carry an absolute Root and Script — a test may hand either a copy +// — and the repository-relative CIRef, because that is the string a workflow can +// actually name. +func stackPaths(t *testing.T) (workflow, root string, sources []exampleSource) { t.Helper() repo := repoRoot(t) - return filepath.Join(repo, conformanceWorkflow), - filepath.Join(repo, conformanceRoot), - filepath.Join(repo, stacksRoot), - filepath.Join(repo, stacksScript) + for _, source := range exampleSources() { + sources = append(sources, exampleSource{ + Family: source.Family, + Root: filepath.Join(repo, source.Root), + Script: filepath.Join(repo, source.Script), + CIRef: source.CIRef, + }) + } + return filepath.Join(repo, conformanceWorkflow), filepath.Join(repo, conformanceRoot), sources +} + +// withRoot answers the source whose CIRef is the given repository-relative +// script, so a test can name one family among the several. +func withRoot(t *testing.T, sources []exampleSource, ciRef string) exampleSource { + t.Helper() + for _, source := range sources { + if source.CIRef == ciRef { + return source + } + } + t.Fatalf("no example source runs %s: this test is measuring a list it does not understand", ciRef) + return exampleSource{} } // The population both checks judge is not empty, and both halves of the // judgement run here. // -// stackProofProblems answers nothing when examples/stacks is absent, because +// stackProofProblems answers nothing when the example roots are absent, because // `feint docs` also regenerates the README of somebody who installed the binary // and has no such directory. That tolerance is exactly the shape of a check that // stops measuring when its subject moves, so the subject is asserted here rather // than assumed: this repository has stacks, CI applies some of them, and the // list of exceptions is smaller than the list of stacks. func TestTheStackChecksHaveASubjectToMeasure(t *testing.T) { - workflow, _, stacks, script := stackPaths(t) + workflow, _, sources := stackPaths(t) - dirs, err := stackDirs(stacks) - if err != nil { - t.Fatalf("list the stacks: %v", err) + total := 0 + for _, source := range sources { + dirs, err := stackDirs(source.Root) + if err != nil { + t.Fatalf("list the examples under %s: %v", source.Root, err) + } + total += len(dirs) } - if len(dirs) < 3 { - t.Fatalf("only %d stack(s) under %s: the listing is broken, not the stacks", len(dirs), stacksRoot) + if total < 4 { + t.Fatalf("only %d example director(ies) across %d roots: the listing is broken, not the "+ + "examples", total, len(sources)) } - applied, err := stacksAppliedInCI(script, workflow) + applied, err := stacksAppliedInCI(sources, workflow) if err != nil { t.Fatalf("read which stacks CI applies: %v", err) } if len(applied) == 0 { - t.Fatal("no `run_stack` line is read from tools/conformance/stacks.sh: every stack would " + + t.Fatal("no `run_stack` line is read from any example suite: every stack would " + "need a declaration, which is a check measuring its own reader") } - if len(stacksRunByHand) >= len(dirs) { - t.Fatalf("%d of %d stacks are declared as run by hand: an exception list as long as the "+ - "population is not an exception list", len(stacksRunByHand), len(dirs)) + // Both families are really in the population. Adding the quickstart root and + // having nothing under it read would be the same check with a longer list. + for _, source := range sources { + found := false + for dir := range applied { + if strings.HasPrefix(dir, source.Family+"/") { + found = true + break + } + } + if !found { + t.Errorf("%s is declared as an example root and CI applies nothing under it: the "+ + "refusals below would judge a family nobody runs", source.Family) + } + } + if len(stacksRunByHand) >= total { + t.Fatalf("%d of %d examples are declared as run by hand: an exception list as long as the "+ + "population is not an exception list", len(stacksRunByHand), total) } } @@ -60,13 +101,14 @@ func TestTheStackChecksHaveASubjectToMeasure(t *testing.T) { // tools/conformance/stacks.sh, and the generated table prints `no` for it as if // that were a decision. func TestAStackCIDoesNotApplyIsDeclaredWithAReason(t *testing.T) { - workflow, _, stacks, script := stackPaths(t) + workflow, _, sources := stackPaths(t) - if problems := undeclaredStacks(stacks, script, workflow); len(problems) != 0 { + if problems := undeclaredStacks(sources, workflow); len(problems) != 0 { t.Fatalf("the repository does not satisfy its own rule:\n %s", strings.Join(problems, "\n ")) } - body, err := os.ReadFile(script) + stacks := withRoot(t, sources, stacksScript) + body, err := os.ReadFile(stacks.Script) if err != nil { t.Fatal(err) } @@ -81,7 +123,13 @@ func TestAStackCIDoesNotApplyIsDeclaredWithAReason(t *testing.T) { t.Fatal(err) } - problems := undeclaredStacks(stacks, copied, workflow) + loosened := append([]exampleSource{}, sources...) + for i := range loosened { + if loosened[i].CIRef == stacksScript { + loosened[i].Script = copied + } + } + problems := undeclaredStacks(loosened, workflow) if len(problems) == 0 { t.Fatal("a stack no `run_stack` line applies and nothing declares passed: the table " + "would print `no` for it and read as a decision") @@ -95,13 +143,55 @@ func TestAStackCIDoesNotApplyIsDeclaredWithAReason(t *testing.T) { // applied because X" are different facts. restore := stacksRunByHand t.Cleanup(func() { stacksRunByHand = restore }) - stacksRunByHand = []stackException{{Stack: "exoscale", Reason: " "}} - problems = undeclaredStacks(stacks, script, workflow) + stacksRunByHand = []stackException{{Root: stacks.Family, Stack: "exoscale", Reason: " "}} + problems = undeclaredStacks(sources, workflow) if len(problems) == 0 { t.Fatal("a declaration with an empty reason excused a stack anyway") } } +// The same rule reaches the quickstart family, which is the whole reason +// exampleSources is a list (#593). +// +// A quickstart nobody applies is the README rotting, and it is the example most +// readers run. The mutation is the one somebody will really make: a quickstart +// directory added and never wired into its suite. +func TestAQuickstartCIDoesNotApplyIsRefusedToo(t *testing.T) { + workflow, _, sources := stackPaths(t) + quickstart := withRoot(t, sources, quickstartScript) + + body, err := os.ReadFile(quickstart.Script) + if err != nil { + t.Fatal(err) + } + trimmed := strings.Replace(string(body), "run_stack "+quickstartLead, + "true # run_stack "+quickstartLead, 1) + if trimmed == string(body) { + t.Fatalf("%s no longer applies the %s quickstart: either it was removed and a declaration "+ + "must have appeared, or this test is measuring a file it does not understand", + quickstartScript, quickstartLead) + } + copied := filepath.Join(t.TempDir(), "quickstart.sh") + if err := os.WriteFile(copied, []byte(trimmed), 0o600); err != nil { + t.Fatal(err) + } + loosened := append([]exampleSource{}, sources...) + for i := range loosened { + if loosened[i].CIRef == quickstartScript { + loosened[i].Script = copied + } + } + problems := undeclaredStacks(loosened, workflow) + if len(problems) == 0 { + t.Fatalf("the %s quickstart is applied by nothing and declared by nothing, and it passed: "+ + "the first door of this project would rot with no gate saying so", quickstartLead) + } + if !strings.Contains(strings.Join(problems, "\n"), quickstartRoot+"/"+quickstartLead) { + t.Errorf("the refusal names neither the root nor the example:\n %s", + strings.Join(problems, "\n ")) + } +} + // A declaration that excuses nothing is stale, in both the ways it can be. // // This is the half that keeps the list from becoming a place reasons go to be @@ -110,13 +200,14 @@ func TestAStackCIDoesNotApplyIsDeclaredWithAReason(t *testing.T) { // describes, which is the defect CLAUDE.md names as the most expensive one // measured on this repository. func TestADeclarationThatExcusesNothingIsStale(t *testing.T) { - workflow, _, stacks, script := stackPaths(t) + workflow, _, sources := stackPaths(t) + stacks := withRoot(t, sources, stacksScript) restore := stacksRunByHand t.Cleanup(func() { stacksRunByHand = restore }) stacksRunByHand = append(append([]stackException{}, restore...), - stackException{Stack: "kubernetes", Reason: "a stack that was never here"}) - problems := undeclaredStacks(stacks, script, workflow) + stackException{Root: stacks.Family, Stack: "kubernetes", Reason: "a stack that was never here"}) + problems := undeclaredStacks(sources, workflow) if len(problems) == 0 || !strings.Contains(strings.Join(problems, "\n"), "kubernetes") { t.Errorf("a declaration for a stack that does not exist was accepted:\n %s", strings.Join(problems, "\n ")) @@ -125,12 +216,22 @@ func TestADeclarationThatExcusesNothingIsStale(t *testing.T) { // The other direction, and the one that reads like evidence while being // false: the stack is applied on every pull request and the list still says // it is run by hand. - stacksRunByHand = []stackException{{Stack: "scaleway", Reason: "stale: CI applies it"}} - problems = undeclaredStacks(stacks, script, workflow) + stacksRunByHand = []stackException{{Root: stacks.Family, Stack: "scaleway", Reason: "stale: CI applies it"}} + problems = undeclaredStacks(sources, workflow) if len(problems) == 0 || !strings.Contains(strings.Join(problems, "\n"), "scaleway") { t.Errorf("a declaration survived CI starting to apply the stack it excuses:\n %s", strings.Join(problems, "\n ")) } + + // And the third way, which only exists because there are two roots: a + // declaration naming a family nothing walks. Keyed by name alone it would + // have silently excused the stack of the same name in the other root. + stacksRunByHand = []stackException{{Root: "examples/nowhere", Stack: "scaleway", Reason: "a root nothing reads"}} + problems = undeclaredStacks(sources, workflow) + if len(problems) == 0 || !strings.Contains(strings.Join(problems, "\n"), "examples/nowhere") { + t.Errorf("a declaration under a root no source names was accepted:\n %s", + strings.Join(problems, "\n ")) + } } // A stack CI applies names the provider versions it accepts. @@ -143,9 +244,9 @@ func TestADeclarationThatExcusesNothingIsStale(t *testing.T) { // The constraint does not have to be exact — the page says plainly what a floor // is worth — it has to exist. func TestAStackAppliedInCIPinsTheProviderThatAnswered(t *testing.T) { - workflow, root, stacks, script := stackPaths(t) + workflow, root, sources := stackPaths(t) - pins, err := providerPinsOfRepository(workflow, root, stacks, script) + pins, err := providerPinsOfRepository(workflow, root, sources) if err != nil { t.Fatalf("read the provider constraints: %v", err) } @@ -168,6 +269,20 @@ func TestAStackAppliedInCIPinsTheProviderThatAnswered(t *testing.T) { t.Fatalf("a stack CI applies pins nothing:\n %s", strings.Join(problems, "\n ")) } + // The quickstart examples are in that population rather than beside it: the + // first thing a reader applies must pin the provider that answered as much + // as the qualification stack does. + quickstartPins := 0 + for _, pin := range pins { + if strings.Contains(pin.Dir, quickstartRoot+"/") && pin.Driven { + quickstartPins++ + } + } + if quickstartPins == 0 { + t.Errorf("no applied provider entry under %s: the quickstart is outside the population "+ + "this check judges, which is the hole it exists to close", quickstartRoot) + } + // The refusing half, on the entry that really was like this until the // table of #325 named it: the module's constraint removed, nothing else. loosened := make([]providerPin, len(pins)) @@ -197,8 +312,8 @@ func TestAStackAppliedInCIPinsTheProviderThatAnswered(t *testing.T) { func TestThePageCarriesTheReasonAStackIsNotApplied(t *testing.T) { rendered := provedPage(t) for _, e := range stacksRunByHand { - if !strings.Contains(rendered, "`"+stacksRoot+"/"+e.Stack+"` —") { - t.Errorf("the page prints no reason for %s/%s:\n%s", stacksRoot, e.Stack, rendered) + if !strings.Contains(rendered, "`"+e.Root+"/"+e.Stack+"` —") { + t.Errorf("the page prints no reason for %s/%s:\n%s", e.Root, e.Stack, rendered) } } // And the reason is the declared one rather than a second wording of it. diff --git a/internal/cli/docs_write_order_test.go b/internal/cli/docs_write_order_test.go new file mode 100644 index 00000000..59536aad --- /dev/null +++ b/internal/cli/docs_write_order_test.go @@ -0,0 +1,64 @@ +package cli_test + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// One run that changes two sections of the same page keeps both. +// +// `feint docs` writes the target from a copy it spliced at the top of the run, +// and several helpers below it re-read that same file and splice into what they +// find — the safety banner, the quick start, and the promise added by #592. +// Written last, the target's copy is one taken *before* those helpers ran, so it +// puts their sections back the way they were and the run reports success. +// +// Measured while adding the promise block: a single `feint docs` wrote the new +// promise and then reverted it, and the only symptom was `docs --check` still +// red after a regeneration that had said `README.md updated`. Nothing lied; the +// last writer won. +// +// The two markers are chosen so this test needs nothing but the coverage +// artefacts: the coverage table is spliced into the target's own copy, and the +// promise is written by a helper that re-reads the file. Move the target's write +// back below the helpers and this test fails on the promise. +func TestARunThatChangesTwoSectionsOfTheREADMEKeepsBoth(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "README.md") + stale := "intro\n\n" + + "\nwritten by hand, and wrong\n\n\n" + + "\nalso written by hand, and also wrong\n\n\n" + + "outro\n" + if err := os.WriteFile(target, []byte(stale), 0o600); err != nil { + t.Fatal(err) + } + + coverage := repoPath(t, "coverage") + if code, _, errOut := run("docs", "--file", target, "--coverage", coverage); code != 0 { + t.Fatalf("docs exited %d: %s", code, errOut) + } + + written, err := os.ReadFile(target) //nolint:gosec // a path this test just made + if err != nil { + t.Fatal(err) + } + body := string(written) + + // Both sections were regenerated, and the assertion is on the stale text + // rather than on the new: a section that vanished entirely would satisfy + // "the new text is there" for the other one and nothing else. + if strings.Contains(body, "written by hand, and wrong") { + t.Error("the coverage section was not regenerated at all") + } + if strings.Contains(body, "also written by hand, and also wrong") { + t.Error("the promise was written and then reverted by the target's own write: one run " + + "changed two sections and kept one, which reads as success and is not") + } + + // And the run is idempotent, which is what a gate reads. + if code, _, errOut := run("docs", "--file", target, "--coverage", coverage, "--check"); code != 0 { + t.Fatalf("docs --check is red right after a write: %s", errOut) + } +} diff --git a/mise.toml b/mise.toml index b7fdbd91..9589b570 100644 --- a/mise.toml +++ b/mise.toml @@ -235,6 +235,20 @@ run = [ description = "Apply the example stacks — realistic platform configurations, not fixtures" run = "tools/conformance/stacks.sh http://$FEINT_ADDR" +[tasks."conformance:quickstart"] +description = "Run the README's own four commands: `feint up` on examples/quickstart, an empty second plan, `feint down`" +depends = ["build"] +# On a port of its own, because this suite starts and stops emulators of its +# own: it is `feint up` and `feint down` that are under test, and pointed at the +# shared address its cleanup would stop the process every other suite is +# measuring. Same reasoning as the environment suite, faults.sh and zones.sh. +# +# It exists because #593 asks for the thing that keeps a first example from +# rotting. The qualification stacks have had a gate since the day they landed; +# the example the most people copy had none, and it is the one whose breakage a +# reader meets before anything else. +run = "tools/conformance/quickstart.sh 127.0.0.1:4597" + [tasks."conformance:environment"] description = "Prove `feint up` and `feint down` on a fixture declaration, with the real binary" depends = ["build"] @@ -562,6 +576,11 @@ tools/conformance/outscale/terraform.sh "http://$FEINT_ADDR" # platform team writes Terraform rather than the way a fixture is written, and # they found two defects every other gate was blind to (#249, #250). tools/conformance/stacks.sh "http://$FEINT_ADDR" +# And the other family of examples: the README's own four commands, on an +# emulator and a port of their own because `feint up` and `feint down` are what +# they drive. Small, so it costs seconds; gated, so the first thing a reader +# copies cannot rot unnoticed (#593). +tools/conformance/quickstart.sh 127.0.0.1:4597 # The leg that is not here, named so its absence reads as the decision it is: # no Terraform for Exoscale until upstream #573 is fixed (#525 — five signed # requests left for api-ch-*.exoscale.com from a `feint down`). The exo legs diff --git a/tools/conformance/quickstart.sh b/tools/conformance/quickstart.sh new file mode 100755 index 00000000..c171c596 --- /dev/null +++ b/tools/conformance/quickstart.sh @@ -0,0 +1,209 @@ +#!/usr/bin/env bash +# The quick start, run exactly the way the README tells a reader to run it. +# +# #593 asks for a first example short enough to read, and then for the thing +# that keeps it from rotting: *a quickstart nobody runs is a README that rots*, +# and this is the example the most people will copy. So it gets a gate, and the +# gate is cheap precisely because the example is small. +# +# What it asserts, and the third is the one that finds things: +# +# 1. `feint up` on the example's own directory reaches a state where the +# emulator answers and every declared ready condition passed — which is the +# four-command journey the README prints, not an approximation of it; +# 2. the emulator itself answers afterwards, read from the emulator rather +# than from Terraform's state file; +# 3. **the second plan is empty** — where an emulator that answers 200 and +# stores something else shows up. Both defects the qualification stacks +# found (#249, #250) were found by that plan rather than by an apply. +# +# Then `feint down`, and nothing answering the port afterwards. +# +# **No machine runtime, ever.** The examples declare `runtime: mode: off` and +# this script never overrides it: thirty seconds means a control plane, an +# address and a server the API describes. `FEINT_VM` is deliberately not read +# here — the runtime depth has examples/stacks and its own suites. +# +# It carries no proof.json and must not grow one. #503's rule is that a family a +# stack declares must be asserted and a family it cannot offer must write its +# reason; a quickstart that boots no machine can honour none of the families +# tools/conformance/functional.sh asks about, so declaring them would be a proof +# it cannot honour. functional.sh reads examples/stacks and only that. +# +# Usage: tools/conformance/quickstart.sh [addr] +set -uo pipefail + +# Its own port by default, never the shared one: this suite starts and stops an +# emulator of its own, and pointing it at the address the rest of a run shares +# would have its cleanup stop somebody else's process. Same reasoning as +# tools/conformance/environment/up.sh, faults.sh and zones.sh. +ADDR="${1:-127.0.0.1:4597}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" +FEINT="${FEINT_BIN:-$ROOT/feint}" + +fail() { echo "FAIL: $*" >&2; exit 1; } +ok() { echo " ok: $*"; } + +# Never let a client reach anything but the local emulator. Without this, a +# missing endpoint does not fail: every official client falls back to the +# operator's stored credentials, and a test creates billable resources on a real +# account. That is not hypothetical — it happened, to this repository. +# shellcheck source=/dev/null +. "$SCRIPT_DIR/guard.sh" +guard_local "http://$ADDR" + +WORK="" +EXAMPLE="" +cleanup() { + # The emulator goes even when an assertion failed: a leftover process holds + # the port, and the next run then measures it instead of the code under test. + # `down` first, so a failed run still destroys what it created; `stop` after, + # because `down` is the verb that can refuse. + if [ -n "$WORK" ] && [ -f "$WORK/terraform.tfstate" ]; then + (cd "$WORK" && "$FEINT" down >/dev/null 2>&1) \ + || echo "warning: could not take $EXAMPLE down; the stop below still runs" >&2 + fi + "$FEINT" stop --addr "$ADDR" >/dev/null 2>&1 + [ -n "$WORK" ] && rm -rf "$WORK" + WORK="" +} +# EXIT rather than RETURN, and the difference is measured: fail() exits the +# script, and bash runs a RETURN trap only when a function returns. With RETURN +# the first broken run leaves the emulator holding the port, and every later run +# reports its leftovers instead of the defect that stopped the first. +trap cleanup EXIT INT TERM + +[ -x "$FEINT" ] || fail "no feint binary at $FEINT; run \`mise run build\` first" + +echo "conformance: the quick start examples, run the way the README prints them, on $ADDR" + +# run_stack is the same verb tools/conformance/stacks.sh uses, and the spelling +# matters beyond taste: internal/cli/docs_proved.go reads `run_stack ` +# lines to answer which examples CI applies, and `feint docs --check` refuses an +# example directory no such line names. One idiom, one reader. +run_stack() { # name + local name="$1" + local src="$ROOT/examples/quickstart/$name" + [ -d "$src" ] || fail "no quickstart at $src" + + # A copy, so the repository nobody asked to dirty stays clean: `up` records an + # instance and the engine writes state beside the declaration. *.tf and + # feint.yaml explicitly, never the whole directory: a reader who ran the + # example in place leaves .terraform/ and terraform.tfstate behind, and + # copying those would hand this run somebody else's state. + WORK="$(mktemp -d)" + EXAMPLE="$name" + cp "$src"/*.tf "$WORK/" || fail "$name: cannot copy the configuration" + cp "$src/feint.yaml" "$WORK/" || fail "$name: cannot copy the declaration" + sed -i "s|addr: 127.0.0.1:4599|addr: $ADDR|g; s|tcp:127.0.0.1:4599|tcp:$ADDR|g" "$WORK/feint.yaml" + cd "$WORK" || fail "$name: cannot enter the work directory" + export TF_IN_AUTOMATION=1 TF_INPUT=0 + + echo "- $name: feint up" + local log="$WORK/up.log" + "$FEINT" up --timeout 120s >"$log" 2>&1 + local code=$? + [ "$code" -eq 0 ] || { cat "$log"; fail "$name: up exited $code"; } + ok "up exited 0" + + # Every condition the declaration names was said out loud and confirmed. A + # green run that printed nothing would be indistinguishable from one that + # skipped the wait. The list is read from the declaration rather than restated + # here, so an example that adds a condition is checked on it the same day. + local conditions condition + conditions="$(sed -n 's/^ *- \(http:[^ ]*\|tcp:[^ ]*\|resource:[^ ]*\)$/\1/p' "$WORK/feint.yaml")" + [ -n "$conditions" ] || fail "$name: feint.yaml declares no ready condition, so up proved nothing" + while IFS= read -r condition; do + [ -n "$condition" ] || continue + grep -qF "ok: $condition" "$log" \ + || { cat "$log"; fail "$name: the ready condition $condition was never confirmed"; } + done <<<"$conditions" + ok "every declared ready condition was said out loud and confirmed" + + # Asserted against the emulator rather than against up's own output. + curl -sf "http://$ADDR/_feint/health" >/dev/null \ + || fail "$name: the emulator up brought up does not answer" + ok "the emulator answers" + + # And the line the README prints under those four commands is the line this + # run produced. + # + # #593's second finding was that it was not: the block showed `Apply complete! + # Resources: 5 added` above three commands with no directory, no `main.tf` and + # no provider block, so a reader following the quick start exactly could not + # arrive at the output the quick start displayed. The number is derived now + # (internal/cli/docs_quickstart.go counts the resource blocks), and this is the + # other half: the sentence is lifted out of the README and required verbatim, + # so a generated claim about output is checked against output. + # + # Only for the example the README documents, which is read from the block's own + # `cd` line rather than assumed: the README leads with one pack and this suite + # runs every one of them. + local readme="$ROOT/README.md" + if [ -f "$readme" ]; then + local documented printed + documented="$(sed -n 's|^cd [^/]*/examples/quickstart/\([a-z0-9-]*\)$|\1|p' "$readme" | head -1)" + printed="$(grep -m1 '^Apply complete! Resources:' "$readme")" + if [ "$documented" = "$name" ]; then + [ -n "$printed" ] \ + || fail "$name: the README documents this example and prints no apply line, so nothing says what the four commands produce" + grep -qF "$printed" "$log" \ + || { cat "$log"; fail "$name: the README prints \"$printed\" and this run did not: the output shown is not the output produced"; } + ok "the README's own apply line is the one this run printed" + fi + fi + + # The assertion that separates a test from a demonstration: what was sent is + # what is served back. No environment is exported for it, and that is not an + # omission — both examples carry their credentials and their endpoint in the + # configuration, which is what makes them copyable in the first place. + # + # The engine is the one the declaration names, resolved the way `feint up` + # resolves it: exec.LookPath on `iac.engine`, so the binary that plans is the + # binary that applied. The first version of this script guessed instead — + # `tofu` if installed, else `terraform` — and on a station holding both it + # planned with OpenTofu a directory Terraform had initialised. What comes back + # is not a drift report but *Inconsistent dependency lock file*, because the + # lock names `registry.terraform.io` and OpenTofu wants + # `registry.opentofu.org`: exit status 1, and a case arm reading "the second + # plan errored" about an emulator that had answered perfectly. Measured + # 2026-08-28, on the first real run of this gate. + local engine + engine="$(sed -n 's/^ *engine: *\([a-z]*\) *$/\1/p' "$WORK/feint.yaml" | head -1)" + [ -n "$engine" ] || fail "$name: feint.yaml declares no iac.engine, so nothing says what applied" + command -v "$engine" >/dev/null 2>&1 \ + || fail "$name: the declaration names the engine $engine and it is not installed" + + echo "- $name: the second plan is empty, read by $engine" + "$engine" plan -no-color -detailed-exitcode -var "endpoint=http://$ADDR" >/dev/null 2>&1 + local status=$? + case "$status" in + 0) ok "no drift between what was sent and what is served" ;; + 2) "$engine" plan -no-color -var "endpoint=http://$ADDR" || true + fail "$name: the emulator does not read back what the quick start sent" ;; + *) "$engine" plan -no-color -var "endpoint=http://$ADDR" || true + fail "$name: the second plan errored with status $status" ;; + esac + + echo "- $name: feint down" + "$FEINT" down >"$WORK/down.log" 2>&1 + code=$? + [ "$code" -eq 0 ] || { cat "$WORK/down.log"; fail "$name: down exited $code"; } + ok "down exited 0" + + # Checked, not assumed. + if curl -sf --max-time 2 "http://$ADDR/_feint/health" >/dev/null 2>&1; then + fail "$name: down returned and something still answers on $ADDR" + fi + ok "nothing answers on $ADDR" + + cd "$ROOT" || fail "$name: cannot leave the work directory" + rm -rf "$WORK" + WORK="" +} + +run_stack scaleway +run_stack outscale + +echo "conformance: the quick start examples were brought up, re-planned empty and taken down" diff --git a/tools/falsify/specs/capability-matrix.json b/tools/falsify/specs/capability-matrix.json new file mode 100644 index 00000000..cbacee2f --- /dev/null +++ b/tools/falsify/specs/capability-matrix.json @@ -0,0 +1,96 @@ +{ + "package": "./internal/cli/", + "mutations": [ + { + "label": "the matrix stops asking the pack whether it vetoes the engine, so a row may promise Terraform on a pack `feint up` refuses before a process starts — #592, moved one storey down into the table", + "file": "internal/cli/capability.go", + "find": "\t\tif engine != \"\" && vetoes[row.Provider][engine] != \"\" {", + "replace": "\t\tif false && engine != \"\" && vetoes[row.Provider][engine] != \"\" {", + "test": "TestTheMatrixAndTheDoorstepCannotDisagree" + }, + { + "label": "a veto no row carries stops being reported, so the refusal lives in up.go and no generated sentence can mention it", + "file": "internal/cli/capability.go", + "find": "\t\t\tif claimed[key] {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tproblems = append(problems, fmt.Sprintf(\n\t\t\t\t\"the %s pack vetoes %s and capabilityMatrix has no row for it: the refusal exists in \"+", + "replace": "\t\t\tif claimed[key] || true {\n\t\t\t\tcontinue\n\t\t\t}\n\t\t\tproblems = append(problems, fmt.Sprintf(\n\t\t\t\t\"the %s pack vetoes %s and capabilityMatrix has no row for it: the refusal exists in \"+", + "test": "TestTheMatrixAndTheDoorstepCannotDisagree" + }, + { + "label": "a supported row stops being checked against the workflow that is supposed to prove it, so `proof` becomes decoration", + "file": "internal/cli/capability.go", + "find": "\t\tif !drives[key] {", + "replace": "\t\tif !drives[key] && false {", + "test": "TestASupportedRowNamesAWorkflowThatDrivesIt" + }, + { + "label": "a pair CI drives with no row at all passes, so a client this project really proves can be written down nowhere — the understated half of the same defect", + "file": "internal/cli/capability.go", + "find": "\tfor key := range drives {\n\t\tif claimed[key] {", + "replace": "\tfor key := range drives {\n\t\tif claimed[key] || true {", + "test": "TestASupportedRowNamesAWorkflowThatDrivesIt" + }, + { + "label": "a refused row may name no reason and no marker, so \"refused\" stops being a decision and no sentence could name the pair at all", + "file": "internal/cli/capability.go", + "find": "\t\tcase capabilityRefused:\n\t\t\tif strings.TrimSpace(row.Reason) == \"\" {", + "replace": "\t\tcase capabilityRefused:\n\t\t\tif strings.TrimSpace(row.Reason) == \"\" && false {", + "test": "TestARefusedRowCarriesItsReasonAndItsMarker" + }, + { + "label": "the mode column stops being read from the run it describes, so every row claims the control plane whatever the workflow arms", + "file": "internal/cli/capability.go", + "find": "\t\tif m[1] == \"off\" {", + "replace": "\t\tif m[1] == \"off\" || true {", + "test": "TestTheModeColumnIsReadFromTheRunItDescribes" + }, + { + "label": "a claim naming a pair the matrix carries no row for at all passes, so an unowned capability claim becomes writable again — the class of README.md:41, in the half where nothing was ever declared", + "file": "internal/cli/capability_claims.go", + "find": "\t\t\t\tcase row == nil:", + "replace": "\t\t\t\tcase row == nil && false:", + "test": "TestTheClaimReaderRefusesEveryShapeOfTheClaim" + }, + { + "label": "a sentence claiming a refused pair passes without naming what would change it, so the README can promise the one client the doorstep turns away", + "file": "internal/cli/capability_claims.go", + "find": "\t\t\t\tcase row.Support == capabilityRefused &&", + "replace": "\t\t\t\tcase false && row.Support == capabilityRefused &&", + "test": "TestTheClaimReaderRefusesEveryShapeOfTheClaim" + }, + { + "label": "a list item stops being its own unit, so two bullets read as one paragraph and `scw` is claimed for the pack the next bullet names", + "file": "internal/cli/capability_claims.go", + "find": "\t\tif bulletStart.MatchString(trimmed) {", + "replace": "\t\tif bulletStart.MatchString(trimmed) && false {", + "test": "TestAUnitIsASentenceATableRowAListItemOrAWholeFence" + }, + { + "label": "a table row stops being its own unit, so a table pairs every client it lists with every provider it lists", + "file": "internal/cli/capability_claims.go", + "find": "\t\tif strings.HasPrefix(trimmed, \"|\") {", + "replace": "\t\tif strings.HasPrefix(trimmed, \"|\") && false {", + "test": "TestAUnitIsASentenceATableRowAListItemOrAWholeFence" + }, + { + "label": "the quickstart family leaves the population, so an example nobody applies is no longer a finding — the README rotting with no gate saying so", + "file": "internal/cli/docs_proved.go", + "find": "\t\t{Family: quickstartRoot, Root: quickstartRoot, Script: quickstartScript, CIRef: quickstartScript},", + "replace": "\t\t{Family: quickstartRoot, Root: quickstartRoot, Script: quickstartScript, CIRef: \"tools/conformance/nothing-runs-this.sh\"},", + "test": "TestAQuickstartCIDoesNotApplyIsRefusedToo" + }, + { + "label": "a quick start whose resource blocks are not its objects is counted anyway, so the front page prints an apply line no run produces — #593's second finding, restored", + "file": "internal/cli/docs_quickstart.go", + "find": "\t\tif strings.HasPrefix(trimmed, \"count \") || strings.HasPrefix(trimmed, \"count=\") ||\n\t\t\tstrings.HasPrefix(trimmed, \"for_each \") || strings.HasPrefix(trimmed, \"for_each=\") {", + "replace": "\t\tif false && (strings.HasPrefix(trimmed, \"count \") || strings.HasPrefix(trimmed, \"count=\") ||\n\t\t\tstrings.HasPrefix(trimmed, \"for_each \") || strings.HasPrefix(trimmed, \"for_each=\")) {", + "test": "TestAQuickStartThatMultipliesItsResourcesIsRefusedRatherThanMiscounted" + }, + { + "label": "the target is written last again, so a run that changes two sections of the README keeps one and reports success", + "file": "internal/cli/docs.go", + "find": "\ttargetChanged := updated != string(current)\n\tif targetChanged {", + "replace": "\ttargetChanged := updated != string(current)\n\tif targetChanged && false {", + "test": "TestARunThatChangesTwoSectionsOfTheREADMEKeepsBoth" + } + ] +} diff --git a/tools/falsify/specs/stack-proof.json b/tools/falsify/specs/stack-proof.json index 8000e15d..9f64aceb 100644 --- a/tools/falsify/specs/stack-proof.json +++ b/tools/falsify/specs/stack-proof.json @@ -11,8 +11,8 @@ { "label": "a stack no run_stack line applies stops needing a declaration, so the table prints `no` for it as if that were a decision", "file": "internal/cli/docs_stacks.go", - "find": "\t\tif _, ok := declared[stack]; !ok {", - "replace": "\t\tif _, ok := declared[stack]; !ok && false {", + "find": "\t\t\tif _, ok := declared[key]; !ok {", + "replace": "\t\t\tif _, ok := declared[key]; !ok && false {", "test": "TestAStackCIDoesNotApplyIsDeclaredWithAReason" }, { @@ -25,15 +25,15 @@ { "label": "a declaration for a stack that no longer exists stops being stale, so the list keeps reasons nothing checks", "file": "internal/cli/docs_stacks.go", - "find": "\t\tif !existing[e.Stack] {", - "replace": "\t\tif !existing[e.Stack] && false {", + "find": "\t\tif !existing[key] {", + "replace": "\t\tif !existing[key] && false {", "test": "TestADeclarationThatExcusesNothingIsStale" }, { "label": "a declaration survives CI starting to apply the stack it excuses, which is the direction that reads like evidence while being false", "file": "internal/cli/docs_stacks.go", - "find": "\t\tif applied[e.Stack] {", - "replace": "\t\tif applied[e.Stack] && false {", + "find": "\t\tif applied[key] {\n\t\t\tproblems = append(problems, fmt.Sprintf(\n\t\t\t\t\"stacksRunByHand says %s is run by hand and %s applies it", + "replace": "\t\tif applied[key] && false {\n\t\t\tproblems = append(problems, fmt.Sprintf(\n\t\t\t\t\"stacksRunByHand says %s is run by hand and %s applies it", "test": "TestADeclarationThatExcusesNothingIsStale" } ], diff --git a/tools/testplan/rules.go b/tools/testplan/rules.go index 1cfef361..adab0529 100644 --- a/tools/testplan/rules.go +++ b/tools/testplan/rules.go @@ -401,6 +401,19 @@ var rules = []rule{ Unproven: "the Exoscale stack is applied by hand, never by CI: no gate here clones a third-party " + "repository and a patched client is not the official one (#525)", }, + { + Path: "examples/quickstart/", + Why: "the first example a reader copies, and the four commands the README prints", + Runs: []string{"conformance:quickstart"}, + Unproven: "the suite drives `feint up` and `feint down` with no machine runtime, which is what " + + "thirty seconds means; a quickstart that booted a machine would need the runtime legs " + + "the qualification stacks have (#593)", + Cites: []claim{{ + About: "the suite drives `feint up` and `feint down` with no machine runtime", + In: "tools/conformance/quickstart.sh", + Shows: []string{"feint up", "feint down", "FEINT_VM` is deliberately not read"}, + }}, + }, { Path: "examples/", Why: "the example declarations and fixtures", @@ -561,6 +574,18 @@ var rules = []rule{ Runs: []string{"FEINT_VM=incus-ovn mise run conformance:functional"}, Unproven: "the unit tests judge each verdict; only the gate judges them against machines that really boot", }, + { + Path: "tools/conformance/quickstart.sh", + Why: "the gate on the four commands the README prints", + Runs: []string{"conformance:quickstart"}, + Unproven: "it judges what a reader copies, never a machine that boots: the examples it applies " + + "declare `runtime: mode: off` and the suite never overrides it (#593)", + Cites: []claim{{ + About: "the examples it applies declare `runtime: mode: off`", + In: "examples/quickstart/scaleway/feint.yaml", + Shows: []string{"mode: off"}, + }}, + }, { Path: "tools/conformance/", Why: "the shared harness: doorstep, score, faults, refusals, stacks, functional",