Run the port examples this site quotes, in a server the site owns

.github/workflows/test.yml

@@ -43,7 +43,7 @@ jobs:
       - name: Install audit tools
         run: |
           sudo apt-get update
-          sudo apt-get install --no-install-recommends -y ripgrep fd-find
+          sudo apt-get install --no-install-recommends -y ripgrep fd-find tmux
           sudo ln -s /usr/bin/fdfind /usr/local/bin/fd
 
       - run: pnpm install --frozen-lockfile

CONTRIBUTING.md

@@ -114,6 +114,23 @@ Missing port checkouts, a missing local server, or a missing `shellcheck`
 can leave publication checks unexercised. Development loops deliberately exclude assembled-output
 suites; they do not establish publication readiness.
 
+Run the port examples these docs quote against a tmux server the docs own:
+
+```console
+$ pnpm test:arena
+```
+
+`scripts/docs-arena.mjs` starts tmux itself with `-D -S` and an empty
+config, lends that server to each port's arena adapter in the
+`<checkout>-tmux-arena` worktree beside the port's checkout, and requires
+one `LIBTMUX_ARENA_EVIDENCE` record naming the server's live challenge, PID,
+and socket. It then withholds the socket and requires the adapter to fail
+without evidence and without reaching any other server. A port without its
+worktree or toolchain is reported as not run; `--require` makes that a
+failure, and `--port <slug>` selects ports. The publication audit runs only
+the supervisor's negative checks, which need tmux alone, unless
+`LIBTMUX_DOCS_ARENA=1`.
+
 Add focused regression coverage for behavior changes and confirm that a new
 check fails when its intended invariant is broken. Root policy-guide edits
 need link, command, and diff review rather than new tests. Content edits

package.json

@@ -14,7 +14,8 @@
     "test:inner": "node scripts/test-loop.mjs inner",
     "test:medium": "node scripts/test-loop.mjs medium",
     "test:fast": "node scripts/test-loop.mjs medium",
-    "test:publication": "./scripts/test-all.sh"
+    "test:publication": "./scripts/test-all.sh",
+    "test:arena": "node scripts/docs-arena.mjs"
   },
   "engines": {
     "node": ">=24"

scripts/arena/check-quote-coverage.mjs

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+/*
+ * Fail on a page that quotes an example program the arena never runs.
+ *
+ * check-quote-drift.mjs asks the other direction — of the sources an artifact
+ * runs, does the page show the same bytes. That leaves the gap this closes: a
+ * page is free to fence a file no artifact has ever executed, and it renders
+ * exactly as well as one that is tested every run. Until now that case was
+ * printed as "quoted with no arena adapter yet" and the run still passed, so
+ * the list of them could grow without anyone deciding to let it.
+ *
+ * A quoted source must therefore be one of two things. Either an artifact runs
+ * it, or it is listed below with a reason code from the shared table, naming
+ * the gate that does run it. An unlisted one fails; so does a listed one that
+ * an artifact now runs, or that no page quotes any more, so the list can only
+ * shrink.
+ *
+ * Reads two data files and nothing else: no worktree, no toolchain, no tmux.
+ * So it runs in the gate every CI job executes, not the port lane — which is
+ * the point, because a page is added in a checkout that has no ports.
+ *
+ * Usage:
+ *   node scripts/arena/check-quote-coverage.mjs
+ *   node scripts/arena/check-quote-coverage.mjs --json
+ */
+import sources from '../../site/src/data/example-sources.json' with { type: 'json' }
+import { ARTIFACTS } from './artifacts.mjs'
+
+/**
+ * The reason codes an exemption may carry, from the shared table the ports and
+ * the site both draw on. `platform:` takes a name, so it is matched by prefix.
+ *
+ * A code outside this set is a failure rather than a new code: the table is
+ * meant to be argued over once and then cited, and a one-off string in a list
+ * like this is how a table stops meaning anything.
+ */
+export const REASON_CODES = new Set([
+  'needs-client',
+  'needs-terminal',
+  'serves-stdio',
+  'unbounded-stream',
+  'test-context',
+  'ambient',
+  'destructive',
+  'no-tmux',
+  'invalid-by-design',
+  'config',
+  'historical',
+  'pseudo',
+])
+
+export const isReasonCode = (code) => REASON_CODES.has(code) || /^platform:\S+$/.test(code)
+
+/**
+ * Quoted sources the arena does not run, why, and what runs them instead.
+ *
+ * `code` cites the shared table. `why` says what about this particular program
+ * makes the arena the wrong gate for it, in terms a reader can check against
+ * the file. `gate` names what does execute it, so "the arena skips it" never
+ * reads as "nothing tests it".
+ */
+export const NOT_IN_THE_ARENA = new Map([
+  [
+    'rs:crates/libtmux/examples/scratch.rs',
+    {
+      code: 'destructive',
+      why: 'its subject is owning a server: it builds one on a socket path it chooses, asserts no session survives the scope, then shuts the server down and unlinks the socket. Lending it a server would stop the server, and the assertion would be about the supervisor\'s own sessions.',
+      gate: 'the rs example runner, which gives it an owned server and matches its output; its row carries no artifact id, which is how that runner spells owned-only.',
+    },
+  ],
+  [
+    'rs:crates/tmux-mcp/examples/readonly.rs',
+    {
+      code: 'serves-stdio',
+      why: 'it serves MCP on stdin and stdout until its peer hangs up. The arena reads its evidence line from stdout, so an adapter here would have to write evidence into the protocol stream the example exists to demonstrate.',
+      gate: 'the rs example runner, which plays the peer: it writes an MCP handshake, waits for `serverInfo`, and closes.',
+    },
+  ],
+])
+
+/**
+ * @param {object} [input]
+ * @param {Record<string, unknown>} [input.quoted] example-sources.json
+ * @param {Iterable<string>} [input.executed] every key some artifact runs
+ * @param {Map<string, {code: string, why: string, gate: string}>} [input.exempt]
+ */
+export function runCheck({
+  quoted = sources,
+  executed = ARTIFACTS.flatMap((entry) => entry.runs),
+  exempt = NOT_IN_THE_ARENA,
+} = {}) {
+  const runs = new Set(executed)
+  const results = []
+
+  for (const key of Object.keys(quoted).sort()) {
+    const listed = exempt.get(key)
+    if (runs.has(key)) {
+      if (listed) {
+        results.push({
+          key,
+          status: 'fail',
+          reason: `listed as ${listed.code} but an arena artifact runs it now — delete the entry`,
+        })
+      } else {
+        results.push({ key, status: 'run', reason: 'an arena artifact runs it' })
+      }
+      continue
+    }
+    if (!listed) {
+      results.push({
+        key,
+        status: 'fail',
+        reason: 'quoted by a page and run by no arena artifact — give it an artifact, or list it in NOT_IN_THE_ARENA with a reason code and the gate that does run it',
+      })
+      continue
+    }
+    if (!isReasonCode(listed.code)) {
+      results.push({ key, status: 'fail', reason: `"${listed.code}" is not one of the shared reason codes` })
+      continue
+    }
+    results.push({ key, status: 'exempt', reason: `${listed.code}: ${listed.gate}` })
+  }
+
+  // An entry outliving the page that justified it. Nobody reads a list of
+  // exemptions looking for the one that no longer applies to anything.
+  for (const key of exempt.keys()) {
+    if (!Object.hasOwn(quoted, key)) {
+      results.push({ key, status: 'fail', reason: 'listed in NOT_IN_THE_ARENA but no page quotes it — delete the entry' })
+    }
+  }
+
+  return results
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) {
+  const results = runCheck()
+  if (process.argv.includes('--json')) {
+    console.log(JSON.stringify(results, null, 2))
+  } else {
+    for (const r of results) console.log(`${r.status.padEnd(7)} ${r.key} — ${r.reason}`)
+  }
+  const tally = (status) => results.filter((r) => r.status === status).length
+  console.log(`\nquote coverage: ${tally('run')} quoted source(s) run by the arena, ${tally('exempt')} exempt, ${tally('fail')} unaccounted for`)
+  if (tally('fail')) process.exitCode = 1
+}

scripts/arena/check-quote-coverage.negative.mjs

@@ -0,0 +1,126 @@
+#!/usr/bin/env node
+/*
+ * Proof that check-quote-coverage.mjs fails, and fails for the stated reason,
+ * on each way a quoted example can go unaccounted for — and that the list of
+ * exemptions can only shrink.
+ *
+ * The controls matter as much as the defects: a check that rejected every
+ * input would satisfy all four failure cases and still be worthless, and this
+ * one has to keep passing the registry the repository actually ships.
+ *
+ * Reads nothing, so it runs wherever the rest of the suite does.
+ */
+import { NOT_IN_THE_ARENA, isReasonCode, runCheck } from './check-quote-coverage.mjs'
+
+let failures = 0
+function expect(name, results, key, wantStatus, fragment) {
+  const got = results.find((r) => r.key === key)
+  const ok = got && got.status === wantStatus && (!fragment || got.reason.includes(fragment))
+  if (ok) console.log(`ok      ${name}`)
+  else {
+    failures += 1
+    console.error(`FAILED  ${name}: ${got ? `got ${got.status} — ${got.reason}` : `no result for ${key}`}`)
+  }
+}
+
+const exempt = (code) => new Map([['port:examples/excused.rs', { code, why: 'why', gate: 'the port runner' }]])
+
+// Control: the registry this repository ships accounts for every quoted
+// source. Without this the four defects below could all pass while the real
+// check was broken.
+const live = runCheck()
+const unaccounted = live.filter((r) => r.status === 'fail')
+if (unaccounted.length === 0) console.log('ok      the shipped registry accounts for every quoted source')
+else {
+  failures += 1
+  console.error(`FAILED  the shipped registry: ${unaccounted.map((r) => `${r.key} — ${r.reason}`).join('; ')}`)
+}
+
+// Control: an artifact runs it, so nothing more is asked of it.
+expect(
+  'a quoted source an artifact runs',
+  runCheck({ quoted: { 'port:examples/run.rs': '' }, executed: ['port:examples/run.rs'], exempt: new Map() }),
+  'port:examples/run.rs',
+  'run',
+)
+
+// Control: quoted, unrun, listed with a code from the table — the case the
+// list exists for.
+expect(
+  'a quoted source listed with a reason code',
+  runCheck({ quoted: { 'port:examples/excused.rs': '' }, executed: [], exempt: exempt('serves-stdio') }),
+  'port:examples/excused.rs',
+  'exempt',
+  'the port runner',
+)
+
+// The defect this check was written for: a page quotes a program nothing in
+// the arena runs, and nobody decided to allow it.
+expect(
+  'a quoted source nothing runs and nothing excuses',
+  runCheck({ quoted: { 'port:examples/new.rs': '' }, executed: [], exempt: new Map() }),
+  'port:examples/new.rs',
+  'fail',
+  'run by no arena artifact',
+)
+
+// The ratchet: an entry that an artifact now runs has to go, or the list
+// becomes a record of things that used to be true.
+expect(
+  'an exemption an artifact now runs',
+  runCheck({
+    quoted: { 'port:examples/excused.rs': '' },
+    executed: ['port:examples/excused.rs'],
+    exempt: exempt('serves-stdio'),
+  }),
+  'port:examples/excused.rs',
+  'fail',
+  'delete the entry',
+)
+
+// The ratchet, the other way: the page went, so the exemption is excusing
+// nothing.
+expect(
+  'an exemption no page quotes',
+  runCheck({ quoted: {}, executed: [], exempt: exempt('serves-stdio') }),
+  'port:examples/excused.rs',
+  'fail',
+  'no page quotes it',
+)
+
+// A reason has to cite the shared table. Free text here would let each
+// exemption invent its own category, which is how a table stops meaning
+// anything.
+expect(
+  'an exemption with an invented code',
+  runCheck({ quoted: { 'port:examples/excused.rs': '' }, executed: [], exempt: exempt('too-slow') }),
+  'port:examples/excused.rs',
+  'fail',
+  'not one of the shared reason codes',
+)
+
+// `platform:` carries a name, so it is matched by prefix rather than listed.
+// Both halves are asserted: a bare `platform` is not a code.
+for (const [code, want] of [['platform:windows', true], ['platform', false], ['platform:', false]]) {
+  if (isReasonCode(code) === want) console.log(`ok      "${code}" ${want ? 'is' : 'is not'} a reason code`)
+  else {
+    failures += 1
+    console.error(`FAILED  "${code}": isReasonCode returned ${isReasonCode(code)}`)
+  }
+}
+
+// Every shipped exemption cites the table, checked directly rather than only
+// through the registry above, so a future entry cannot pass by being unquoted.
+for (const [key, entry] of NOT_IN_THE_ARENA) {
+  if (isReasonCode(entry.code) && entry.gate && entry.why) console.log(`ok      ${key} cites ${entry.code} and names its gate`)
+  else {
+    failures += 1
+    console.error(`FAILED  ${key}: code "${entry.code}", gate "${entry.gate}"`)
+  }
+}
+
+if (failures) {
+  console.error(`quote coverage (negative): ${failures} expectation(s) failed`)
+  process.exit(1)
+}
+console.log('quote coverage (negative): every defect rejected; the controls passed')