diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 63a7eb01..417fde34 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -43,7 +43,7 @@ jobs: - name: Install audit tools run: | sudo apt-get update - sudo apt-get install --no-install-recommends -y ripgrep fd-find + sudo apt-get install --no-install-recommends -y ripgrep fd-find tmux sudo ln -s /usr/bin/fdfind /usr/local/bin/fd - run: pnpm install --frozen-lockfile diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8f3a8d95..ab9a0bed 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -114,6 +114,23 @@ Missing port checkouts, a missing local server, or a missing `shellcheck` can leave publication checks unexercised. Development loops deliberately exclude assembled-output suites; they do not establish publication readiness. +Run the port examples these docs quote against a tmux server the docs own: + +```console +$ pnpm test:arena +``` + +`scripts/docs-arena.mjs` starts tmux itself with `-D -S` and an empty +config, lends that server to each port's arena adapter in the +`-tmux-arena` worktree beside the port's checkout, and requires +one `LIBTMUX_ARENA_EVIDENCE` record naming the server's live challenge, PID, +and socket. It then withholds the socket and requires the adapter to fail +without evidence and without reaching any other server. A port without its +worktree or toolchain is reported as not run; `--require` makes that a +failure, and `--port ` selects ports. The publication audit runs only +the supervisor's negative checks, which need tmux alone, unless +`LIBTMUX_DOCS_ARENA=1`. + Add focused regression coverage for behavior changes and confirm that a new check fails when its intended invariant is broken. Root policy-guide edits need link, command, and diff review rather than new tests. Content edits diff --git a/package.json b/package.json index 5005a326..9807ec3b 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,8 @@ "test:inner": "node scripts/test-loop.mjs inner", "test:medium": "node scripts/test-loop.mjs medium", "test:fast": "node scripts/test-loop.mjs medium", - "test:publication": "./scripts/test-all.sh" + "test:publication": "./scripts/test-all.sh", + "test:arena": "node scripts/docs-arena.mjs" }, "engines": { "node": ">=24" diff --git a/scripts/arena/artifacts.mjs b/scripts/arena/artifacts.mjs new file mode 100644 index 00000000..15c55dea --- /dev/null +++ b/scripts/arena/artifacts.mjs @@ -0,0 +1,515 @@ +/* + * Which port examples the docs arena runs, and how to build and run each one + * from its port's `tmux-arena` worktree. + * + * Each entry names the artifact its adapter accepts and the sources that + * artifact executes, in `site/src/data/example-sources.json` key form + * (`:`), or `:page:` for a port documentation page. + * A quoted source that no entry runs fails check-quote-coverage.mjs unless it + * is listed there with a reason code and the gate that does run it. + * + * `LIBTMUX_DOCS_ARENA_` overrides one port's worktree, which otherwise + * sits beside its checkout as `-tmux-arena`. + */ +import { readFileSync, writeFileSync } from 'node:fs' +import { arch } from 'node:os' +import { join } from 'node:path' +import { PORTS } from '../../site/src/lib/ports.ts' +import { expand } from '../../site/src/plugins/remark-port-code.mjs' + +export function arenaWorktree(slug) { + const override = process.env[`LIBTMUX_DOCS_ARENA_${slug.toUpperCase()}`] + if (override) return expand(override) + const port = PORTS.find((candidate) => candidate.slug === slug) + return port ? `${expand(port.checkout)}-tmux-arena` : '' +} + +// Prints the examples module's runtime classpath, so the example runs as a plain JVM program. +const gradleClasspath = (out) => `gradle.allprojects { p -> + if (p.path == ':examples') { + p.plugins.withId('java') { + def classpath = p.sourceSets.main.runtimeClasspath + def out = new File(${JSON.stringify(out)}) + p.tasks.register('docsArenaClasspath') { + dependsOn p.tasks.named('classes') + doLast { out.text = classpath.asPath } + } + } + } +} +` + +const swiftTriple = () => `${arch() === 'arm64' ? 'aarch64' : 'x86_64'}-unknown-linux-gnu` + +/** + * `prepare(build)` returns build steps; `run(build)` returns the one command + * the arena lends its server to. `build` is a scratch directory for outputs + * that must not land in the worktree. Every `cwd` is relative to the worktree. + */ +export const ARTIFACTS = [ + { + slug: 'py', + artifact: 'python-workspace-setup', + runs: ['py:page:docs/topics/workspace_setup.md'], + tools: ['uv'], + prepare: () => [], + run: () => ({ + cwd: '.', + command: [ + 'uv', 'run', '--frozen', 'python', '-B', '-m', 'pytest', '--reruns=0', '-p', 'no:cacheprovider', '-q', + '--libtmux-arena-target', 'docs/topics/workspace_setup.md', 'docs/topics/workspace_setup.md', + ], + }), + }, + // Two pages on one lend, which is what the per-source evidence is for: the + // supervisor asks for a record per declared page, so a page that collected + // nothing is a failure rather than a quiet pass. + { + slug: 'py', + artifact: 'python-workspace-and-location', + sources: ['docs/topics/workspace_setup.md', 'docs/topics/self_location.md'], + runs: ['py:page:docs/topics/workspace_setup.md', 'py:page:docs/topics/self_location.md'], + tools: ['uv'], + prepare: () => [], + run: () => ({ + cwd: '.', + command: [ + 'uv', 'run', '--frozen', 'python', '-B', '-m', 'pytest', '--reruns=0', '-p', 'no:cacheprovider', '-q', + '--libtmux-arena-target', 'docs/topics/workspace_setup.md', + '--libtmux-arena-target', 'docs/topics/self_location.md', + 'docs/topics/workspace_setup.md', 'docs/topics/self_location.md', + ], + }), + }, + // One entry per site-quoted ts example. They share the port slug, so + // `--port ts` and LIBTMUX_DOCS_ARENA_TS still select all four, and each + // names its own artifact, test file and `runs` key. Install and build steps + // repeat here because every entry declares what it needs; docs-arena runs a + // given command once per worktree, so they are not paid for four times. + { + slug: 'ts', + artifact: 'typescript-quickstart', + runs: ['ts:examples/quickstart/quickstart.ts'], + tools: ['mise'], + prepare: () => [ + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'install', '--frozen-lockfile'] }, + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'run', '--cwd', 'packages/libtmux', 'build'] }, + ], + run: () => ({ cwd: 'examples', command: ['mise', 'exec', '--', 'bun', 'test', '--no-orphans', 'quickstart/quickstart.test.ts'] }), + }, + { + slug: 'ts', + artifact: 'typescript-capture', + runs: ['ts:examples/capture/capture.ts'], + tools: ['mise'], + prepare: () => [ + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'install', '--frozen-lockfile'] }, + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'run', '--cwd', 'packages/libtmux', 'build'] }, + ], + run: () => ({ cwd: 'examples', command: ['mise', 'exec', '--', 'bun', 'test', '--no-orphans', 'capture/capture.test.ts'] }), + }, + { + slug: 'ts', + artifact: 'typescript-agent', + runs: ['ts:examples/agent/agent.ts'], + tools: ['mise'], + prepare: () => [ + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'install', '--frozen-lockfile'] }, + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'run', '--cwd', 'packages/libtmux', 'build'] }, + ], + run: () => ({ cwd: 'examples', command: ['mise', 'exec', '--', 'bun', 'test', '--no-orphans', 'agent/agent.test.ts'] }), + }, + { + slug: 'ts', + artifact: 'typescript-workspace', + runs: ['ts:examples/workspace/workspace.ts'], + tools: ['mise'], + prepare: () => [ + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'install', '--frozen-lockfile'] }, + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'run', '--cwd', 'packages/libtmux', 'build'] }, + // This example imports the published package, not its source. + { cwd: '.', command: ['mise', 'exec', '--', 'bun', 'run', '--cwd', 'packages/workspace', 'build'] }, + ], + run: () => ({ cwd: 'examples', command: ['mise', 'exec', '--', 'bun', 'test', '--no-orphans', 'workspace/workspace.test.ts'] }), + }, + { + slug: 'rs', + artifact: 'rust-inspect', + runs: ['rs:crates/libtmux/examples/inspect.rs'], + tools: ['cargo'], + prepare: (build) => [{ + cwd: '.', + command: ['cargo', 'build', '--locked', '--quiet', '--manifest-path', 'crates/libtmux/Cargo.toml', '--example', 'inspect', '--target-dir', build], + }], + run: (build) => ({ cwd: '.', command: [join(build, 'debug', 'examples', 'inspect')] }), + }, + { + slug: 'rs', + artifact: 'rust-find', + runs: ['rs:crates/libtmux/examples/find.rs'], + tools: ['cargo'], + prepare: (build) => [{ + cwd: '.', + command: ['cargo', 'build', '--locked', '--quiet', '--manifest-path', 'crates/libtmux/Cargo.toml', '--example', 'find', '--target-dir', build], + }], + run: (build) => ({ cwd: '.', command: [join(build, 'debug', 'examples', 'find')] }), + }, + { + slug: 'go', + artifact: 'go-quickstart', + runs: ['go:examples/quickstart/main.go'], + tools: ['go'], + prepare: (build) => [{ cwd: 'examples', command: ['go', 'test', '-c', '-o', join(build, 'quickstart.test'), './quickstart'] }], + run: (build) => ({ cwd: 'examples/quickstart', command: [join(build, 'quickstart.test'), '-test.run=^TestQuickstart$', '-test.count=1'] }), + }, + // The library's own documented Example functions, which the API reference + // shows. They share one lend: the run refuses to stop it and removes only + // what it created, and the evidence comes from TestMain, because `go test` + // diffs an Example's stdout against its `// Output:` comment verbatim. + { + slug: 'go', + artifact: 'go-tmux-examples', + runs: ['go:tmux/example_test.go'], + tools: ['go'], + prepare: (build) => [{ cwd: '.', command: ['go', 'test', '-c', '-o', join(build, 'tmux.test'), './tmux'] }], + run: (build) => ({ cwd: 'tmux', command: [join(build, 'tmux.test'), '-test.run=^Example', '-test.count=1'] }), + }, + // The other example programs, each proved on its own lent server. They + // share the `go` slug, so `--port go` still selects all of them. + { + slug: 'go', + artifact: 'go-environment', + runs: ['go:examples/environment/main.go'], + tools: ['go'], + prepare: (build) => [{ cwd: 'examples', command: ['go', 'test', '-c', '-o', join(build, 'environment.test'), './environment'] }], + run: (build) => ({ cwd: 'examples/environment', command: [join(build, 'environment.test'), '-test.run=^TestEnvironment$', '-test.count=1'] }), + }, + { + slug: 'go', + artifact: 'go-filter-query', + runs: ['go:examples/filter-query/main.go'], + tools: ['go'], + prepare: (build) => [{ cwd: 'examples', command: ['go', 'test', '-c', '-o', join(build, 'filter-query.test'), './filter-query'] }], + run: (build) => ({ cwd: 'examples/filter-query', command: [join(build, 'filter-query.test'), '-test.run=^TestFilterQuery$', '-test.count=1'] }), + }, + { + slug: 'go', + artifact: 'go-control-mode-subscribe', + runs: ['go:examples/control-mode-subscribe/main.go'], + tools: ['go'], + prepare: (build) => [{ cwd: 'examples', command: ['go', 'test', '-c', '-o', join(build, 'control-mode-subscribe.test'), './control-mode-subscribe'] }], + run: (build) => ({ cwd: 'examples/control-mode-subscribe', command: [join(build, 'control-mode-subscribe.test'), '-test.run=^TestControlModeSubscribe$', '-test.count=1'] }), + }, + { + slug: 'go', + artifact: 'go-option-hook-editing', + runs: ['go:examples/option-hook-editing/main.go'], + tools: ['go'], + prepare: (build) => [{ cwd: 'examples', command: ['go', 'test', '-c', '-o', join(build, 'option-hook-editing.test'), './option-hook-editing'] }], + run: (build) => ({ cwd: 'examples/option-hook-editing', command: [join(build, 'option-hook-editing.test'), '-test.run=^TestOptionHookEditing$', '-test.count=1'] }), + }, + { + slug: 'go', + artifact: 'go-planned-build', + runs: ['go:examples/planned-build/main.go'], + tools: ['go'], + prepare: (build) => [{ cwd: 'examples', command: ['go', 'test', '-c', '-o', join(build, 'planned-build.test'), './planned-build'] }], + run: (build) => ({ cwd: 'examples/planned-build', command: [join(build, 'planned-build.test'), '-test.run=^TestPlannedBuild$', '-test.count=1'] }), + }, + { + slug: 'go', + artifact: 'go-snapshot-browser', + runs: ['go:examples/snapshot-browser/main.go'], + tools: ['go'], + prepare: (build) => [{ cwd: 'examples', command: ['go', 'test', '-c', '-o', join(build, 'snapshot-browser.test'), './snapshot-browser'] }], + run: (build) => ({ cwd: 'examples/snapshot-browser', command: [join(build, 'snapshot-browser.test'), '-test.run=^TestSnapshotBrowser$', '-test.count=1'] }), + }, + { + slug: 'go', + artifact: 'go-fast-path', + runs: ['go:examples/fast-path/main.go'], + tools: ['go'], + prepare: (build) => [{ cwd: 'examples', command: ['go', 'test', '-c', '-o', join(build, 'fast-path.test'), './fast-path'] }], + run: (build) => ({ cwd: 'examples/fast-path', command: [join(build, 'fast-path.test'), '-test.run=^TestFastPath$', '-test.count=1'] }), + }, + { + slug: 'go', + artifact: 'go-workspace', + runs: ['go:workspace/example_test.go'], + tools: ['go'], + // This one lives in the workspace module, not the examples module. + prepare: (build) => [{ cwd: 'workspace', command: ['go', 'test', '-c', '-o', join(build, 'workspace.test'), '.'] }], + run: (build) => ({ cwd: 'workspace', command: [join(build, 'workspace.test'), '-test.run=^TestWorkspaceArenaEndpoint$', '-test.count=1'] }), + }, + { + slug: 'java', + artifact: 'java-build-a-workspace', + runs: ['java:examples/src/main/java/io/github/libtmux/examples/BuildAWorkspace.java'], + tools: ['java'], + prepare: (build) => { + const init = join(build, 'classpath.gradle') + writeFileSync(init, gradleClasspath(join(build, 'classpath.txt'))) + return [{ + cwd: '.', + command: ['./gradlew', '--no-daemon', '--quiet', '--no-configuration-cache', '--init-script', init, ':examples:docsArenaClasspath'], + }] + }, + run: (build) => ({ + cwd: '.', + command: ['java', '-cp', readFileSync(join(build, 'classpath.txt'), 'utf8').trim(), 'io.github.libtmux.examples.BuildAWorkspace'], + }), + }, + // The other example programs. The classpath step is identical in each and + // runs once per worktree. + { + slug: 'java', + artifact: 'java-find-panes-running', + runs: ['java:examples/src/main/java/io/github/libtmux/examples/FindPanesRunning.java'], + tools: ['java'], + prepare: (build) => { + const init = join(build, 'classpath.gradle') + writeFileSync(init, gradleClasspath(join(build, 'classpath.txt'))) + return [{ + cwd: '.', + command: ['./gradlew', '--no-daemon', '--quiet', '--no-configuration-cache', '--init-script', init, ':examples:docsArenaClasspath'], + }] + }, + run: (build) => ({ + cwd: '.', + command: ['java', '-cp', readFileSync(join(build, 'classpath.txt'), 'utf8').trim(), 'io.github.libtmux.examples.FindPanesRunning'], + }), + }, + { + slug: 'java', + artifact: 'java-serve-tmux-over-mcp', + runs: ['java:examples/src/main/java/io/github/libtmux/examples/ServeTmuxOverMcp.java'], + tools: ['java'], + prepare: (build) => { + const init = join(build, 'classpath.gradle') + writeFileSync(init, gradleClasspath(join(build, 'classpath.txt'))) + return [{ + cwd: '.', + command: ['./gradlew', '--no-daemon', '--quiet', '--no-configuration-cache', '--init-script', init, ':examples:docsArenaClasspath'], + }] + }, + run: (build) => ({ + cwd: '.', + command: ['java', '-cp', readFileSync(join(build, 'classpath.txt'), 'utf8').trim(), 'io.github.libtmux.examples.ServeTmuxOverMcp'], + }), + }, + { + slug: 'java', + artifact: 'java-watch-pane-output', + runs: ['java:examples/src/main/java/io/github/libtmux/examples/WatchPaneOutput.java'], + tools: ['java'], + prepare: (build) => { + const init = join(build, 'classpath.gradle') + writeFileSync(init, gradleClasspath(join(build, 'classpath.txt'))) + return [{ + cwd: '.', + command: ['./gradlew', '--no-daemon', '--quiet', '--no-configuration-cache', '--init-script', init, ':examples:docsArenaClasspath'], + }] + }, + run: (build) => ({ + cwd: '.', + command: ['java', '-cp', readFileSync(join(build, 'classpath.txt'), 'utf8').trim(), 'io.github.libtmux.examples.WatchPaneOutput'], + }), + }, + { + slug: 'java', + artifact: 'java-watch-what-changes', + runs: ['java:examples/src/main/java/io/github/libtmux/examples/WatchWhatChanges.java'], + tools: ['java'], + prepare: (build) => { + const init = join(build, 'classpath.gradle') + writeFileSync(init, gradleClasspath(join(build, 'classpath.txt'))) + return [{ + cwd: '.', + command: ['./gradlew', '--no-daemon', '--quiet', '--no-configuration-cache', '--init-script', init, ':examples:docsArenaClasspath'], + }] + }, + run: (build) => ({ + cwd: '.', + command: ['java', '-cp', readFileSync(join(build, 'classpath.txt'), 'utf8').trim(), 'io.github.libtmux.examples.WatchWhatChanges'], + }), + }, + { + slug: 'dotnet', + artifact: 'csharp-one-shot', + runs: ['dotnet:examples/LibTmux.Examples/Snippets/OneShot.cs'], + tools: ['dotnet'], + prepare: () => [{ + cwd: '.', + command: ['dotnet', 'build', 'examples/LibTmux.Examples/LibTmux.Examples.csproj', '--configuration', 'Release', '--framework', 'net10.0', '--nologo', '--verbosity', 'quiet'], + }], + run: () => ({ cwd: '.', command: ['dotnet', 'examples/LibTmux.Examples/bin/Release/net10.0/LibTmux.Examples.dll', '--arena', 'csharp-one-shot'] }), + }, + // One artifact per documented snippet file. Every example case can take a + // lent server now, and the flag names any of them; these are the files the + // documentation quotes. `Mcp.ConnectToSelectedSurface` is deliberately not + // here: it starts a separate MCP process that resolves its own socket, so + // it would report evidence for a server it never used. + { + slug: 'dotnet', + artifact: 'csharp-many-commands-one-process', + runs: ['dotnet:examples/LibTmux.Examples/Snippets/Chaining.cs'], + tools: ['dotnet'], + prepare: () => [{ + cwd: '.', + command: ['dotnet', 'build', 'examples/LibTmux.Examples/LibTmux.Examples.csproj', '--configuration', 'Release', '--framework', 'net10.0', '--nologo', '--verbosity', 'quiet'], + }], + run: () => ({ cwd: '.', command: ['dotnet', 'examples/LibTmux.Examples/bin/Release/net10.0/LibTmux.Examples.dll', '--arena', 'csharp-many-commands-one-process'] }), + }, + { + slug: 'dotnet', + artifact: 'csharp-watch-for-window-add', + runs: ['dotnet:examples/LibTmux.Examples/Snippets/ControlMode.cs'], + tools: ['dotnet'], + prepare: () => [{ + cwd: '.', + command: ['dotnet', 'build', 'examples/LibTmux.Examples/LibTmux.Examples.csproj', '--configuration', 'Release', '--framework', 'net10.0', '--nologo', '--verbosity', 'quiet'], + }], + run: () => ({ cwd: '.', command: ['dotnet', 'examples/LibTmux.Examples/bin/Release/net10.0/LibTmux.Examples.dll', '--arena', 'csharp-watch-for-window-add'] }), + }, + { + slug: 'dotnet', + artifact: 'csharp-host-the-tools-yourself', + runs: ['dotnet:examples/LibTmux.Examples/Snippets/Mcp.cs'], + tools: ['dotnet'], + prepare: () => [{ + cwd: '.', + command: ['dotnet', 'build', 'examples/LibTmux.Examples/LibTmux.Examples.csproj', '--configuration', 'Release', '--framework', 'net10.0', '--nologo', '--verbosity', 'quiet'], + }], + run: () => ({ cwd: '.', command: ['dotnet', 'examples/LibTmux.Examples/bin/Release/net10.0/LibTmux.Examples.dll', '--arena', 'csharp-host-the-tools-yourself'] }), + }, + { + slug: 'dotnet', + artifact: 'csharp-show-hierarchy', + runs: ['dotnet:examples/LibTmux.Examples/Snippets/Tour.cs'], + tools: ['dotnet'], + prepare: () => [{ + cwd: '.', + command: ['dotnet', 'build', 'examples/LibTmux.Examples/LibTmux.Examples.csproj', '--configuration', 'Release', '--framework', 'net10.0', '--nologo', '--verbosity', 'quiet'], + }], + run: () => ({ cwd: '.', command: ['dotnet', 'examples/LibTmux.Examples/bin/Release/net10.0/LibTmux.Examples.dll', '--arena', 'csharp-show-hierarchy'] }), + }, + { + slug: 'cxx', + artifact: 'cpp-tour', + runs: ['cxx:examples/01-tour.cpp'], + tools: ['cmake'], + prepare: () => [ + { cwd: '.', command: ['cmake', '--preset', 'cxx-dev'] }, + { cwd: '.', command: ['cmake', '--build', '--preset', 'cxx-dev', '--target', 'libtmux_example_01_tour'] }, + ], + run: () => ({ cwd: '.', command: ['build/cxx-dev/examples/libtmux_example_01_tour'] }), + }, + // The other examples that can borrow. Each names its own artifact and + // builds its own target; the configure step is shared, and docs-arena + // runs a given command once per worktree. + { + slug: 'cxx', + artifact: 'cpp-workspace', + runs: ['cxx:examples/02-workspace.cpp'], + tools: ['cmake'], + prepare: () => [ + { cwd: '.', command: ['cmake', '--preset', 'cxx-dev'] }, + { cwd: '.', command: ['cmake', '--build', '--preset', 'cxx-dev', '--target', 'libtmux_example_02_workspace'] }, + ], + run: () => ({ cwd: '.', command: ['build/cxx-dev/examples/libtmux_example_02_workspace'] }), + }, + { + slug: 'cxx', + artifact: 'cpp-readme', + runs: ['cxx:examples/05-readme.cpp'], + tools: ['cmake'], + prepare: () => [ + { cwd: '.', command: ['cmake', '--preset', 'cxx-dev'] }, + { cwd: '.', command: ['cmake', '--build', '--preset', 'cxx-dev', '--target', 'libtmux_example_05_readme'] }, + ], + run: () => ({ cwd: '.', command: ['build/cxx-dev/examples/libtmux_example_05_readme'] }), + }, + { + slug: 'cxx', + artifact: 'cpp-streaming', + runs: ['cxx:examples/06-streaming.cpp'], + tools: ['cmake'], + prepare: () => [ + { cwd: '.', command: ['cmake', '--preset', 'cxx-dev'] }, + { cwd: '.', command: ['cmake', '--build', '--preset', 'cxx-dev', '--target', 'libtmux_example_06_streaming'] }, + ], + run: () => ({ cwd: '.', command: ['build/cxx-dev/examples/libtmux_example_06_streaming'] }), + }, + { + slug: 'swift', + artifact: 'swift-querying', + runs: ['swift:Examples/Sources/ExampleCode/Querying.swift'], + tools: ['swift'], + prepare: () => [{ + cwd: '.', + command: ['swift', 'build', '--package-path', 'Examples', '--scratch-path', 'Examples/.build/docs-arena', '--build-tests'], + }], + run: () => ({ + cwd: '.', + command: [`Examples/.build/docs-arena/${swiftTriple()}/debug/ExamplesPackageTests.xctest`, '--testing-library', 'swift-testing', '--filter', 'theThreeListings'], + }), + }, + // The other examples the site quotes. Each filters to the one test that + // runs its example against the lent server; the build step is shared. + { + slug: 'swift', + artifact: 'swift-changing', + runs: ['swift:Examples/Sources/ExampleCode/Changing.swift'], + tools: ['swift'], + prepare: () => [{ + cwd: '.', + command: ['swift', 'build', '--package-path', 'Examples', '--scratch-path', 'Examples/.build/docs-arena', '--build-tests'], + }], + run: () => ({ + cwd: '.', + command: [`Examples/.build/docs-arena/${swiftTriple()}/debug/ExamplesPackageTests.xctest`, '--testing-library', 'swift-testing', '--filter', 'theDocumentedSessionIsBuilt'] + }), + }, + { + slug: 'swift', + artifact: 'swift-waiting', + runs: ['swift:Examples/Sources/ExampleCode/Waiting.swift'], + tools: ['swift'], + prepare: () => [{ + cwd: '.', + command: ['swift', 'build', '--package-path', 'Examples', '--scratch-path', 'Examples/.build/docs-arena', '--build-tests'], + }], + run: () => ({ + cwd: '.', + command: [`Examples/.build/docs-arena/${swiftTriple()}/debug/ExamplesPackageTests.xctest`, '--testing-library', 'swift-testing', '--filter', 'documentedWatchSendsTheDifference'] + }), + }, + { + slug: 'swift', + artifact: 'swift-workspaces', + runs: ['swift:Examples/Sources/ExampleCode/Workspaces.swift'], + tools: ['swift'], + prepare: () => [{ + cwd: '.', + command: ['swift', 'build', '--package-path', 'Examples', '--scratch-path', 'Examples/.build/docs-arena', '--build-tests'], + }], + run: () => ({ + cwd: '.', + command: [`Examples/.build/docs-arena/${swiftTriple()}/debug/ExamplesPackageTests.xctest`, '--testing-library', 'swift-testing', '--filter', 'theDocumentedWorkspaceBuilds'] + }), + }, + { + slug: 'swift', + artifact: 'swift-mcp-embedding', + runs: ['swift:Examples/Sources/ExampleCode/MCPEmbedding.swift'], + tools: ['swift'], + prepare: () => [{ + cwd: '.', + command: ['swift', 'build', '--package-path', 'Examples', '--scratch-path', 'Examples/.build/docs-arena', '--build-tests'], + }], + run: () => ({ + cwd: '.', + command: [`Examples/.build/docs-arena/${swiftTriple()}/debug/ExamplesPackageTests.xctest`, '--testing-library', 'swift-testing', '--filter', 'embeddedToolsListPanes'] + }), + }, +] diff --git a/scripts/arena/check-quote-coverage.mjs b/scripts/arena/check-quote-coverage.mjs new file mode 100644 index 00000000..08f1b5ef --- /dev/null +++ b/scripts/arena/check-quote-coverage.mjs @@ -0,0 +1,145 @@ +#!/usr/bin/env node +/* + * Fail on a page that quotes an example program the arena never runs. + * + * check-quote-drift.mjs asks the other direction — of the sources an artifact + * runs, does the page show the same bytes. That leaves the gap this closes: a + * page is free to fence a file no artifact has ever executed, and it renders + * exactly as well as one that is tested every run. Until now that case was + * printed as "quoted with no arena adapter yet" and the run still passed, so + * the list of them could grow without anyone deciding to let it. + * + * A quoted source must therefore be one of two things. Either an artifact runs + * it, or it is listed below with a reason code from the shared table, naming + * the gate that does run it. An unlisted one fails; so does a listed one that + * an artifact now runs, or that no page quotes any more, so the list can only + * shrink. + * + * Reads two data files and nothing else: no worktree, no toolchain, no tmux. + * So it runs in the gate every CI job executes, not the port lane — which is + * the point, because a page is added in a checkout that has no ports. + * + * Usage: + * node scripts/arena/check-quote-coverage.mjs + * node scripts/arena/check-quote-coverage.mjs --json + */ +import sources from '../../site/src/data/example-sources.json' with { type: 'json' } +import { ARTIFACTS } from './artifacts.mjs' + +/** + * The reason codes an exemption may carry, from the shared table the ports and + * the site both draw on. `platform:` takes a name, so it is matched by prefix. + * + * A code outside this set is a failure rather than a new code: the table is + * meant to be argued over once and then cited, and a one-off string in a list + * like this is how a table stops meaning anything. + */ +export const REASON_CODES = new Set([ + 'needs-client', + 'needs-terminal', + 'serves-stdio', + 'unbounded-stream', + 'test-context', + 'ambient', + 'destructive', + 'no-tmux', + 'invalid-by-design', + 'config', + 'historical', + 'pseudo', +]) + +export const isReasonCode = (code) => REASON_CODES.has(code) || /^platform:\S+$/.test(code) + +/** + * Quoted sources the arena does not run, why, and what runs them instead. + * + * `code` cites the shared table. `why` says what about this particular program + * makes the arena the wrong gate for it, in terms a reader can check against + * the file. `gate` names what does execute it, so "the arena skips it" never + * reads as "nothing tests it". + */ +export const NOT_IN_THE_ARENA = new Map([ + [ + 'rs:crates/libtmux/examples/scratch.rs', + { + code: 'destructive', + why: 'its subject is owning a server: it builds one on a socket path it chooses, asserts no session survives the scope, then shuts the server down and unlinks the socket. Lending it a server would stop the server, and the assertion would be about the supervisor\'s own sessions.', + gate: 'the rs example runner, which gives it an owned server and matches its output; its row carries no artifact id, which is how that runner spells owned-only.', + }, + ], + [ + 'rs:crates/tmux-mcp/examples/readonly.rs', + { + code: 'serves-stdio', + why: 'it serves MCP on stdin and stdout until its peer hangs up. The arena reads its evidence line from stdout, so an adapter here would have to write evidence into the protocol stream the example exists to demonstrate.', + gate: 'the rs example runner, which plays the peer: it writes an MCP handshake, waits for `serverInfo`, and closes.', + }, + ], +]) + +/** + * @param {object} [input] + * @param {Record} [input.quoted] example-sources.json + * @param {Iterable} [input.executed] every key some artifact runs + * @param {Map} [input.exempt] + */ +export function runCheck({ + quoted = sources, + executed = ARTIFACTS.flatMap((entry) => entry.runs), + exempt = NOT_IN_THE_ARENA, +} = {}) { + const runs = new Set(executed) + const results = [] + + for (const key of Object.keys(quoted).sort()) { + const listed = exempt.get(key) + if (runs.has(key)) { + if (listed) { + results.push({ + key, + status: 'fail', + reason: `listed as ${listed.code} but an arena artifact runs it now — delete the entry`, + }) + } else { + results.push({ key, status: 'run', reason: 'an arena artifact runs it' }) + } + continue + } + if (!listed) { + results.push({ + key, + status: 'fail', + reason: 'quoted by a page and run by no arena artifact — give it an artifact, or list it in NOT_IN_THE_ARENA with a reason code and the gate that does run it', + }) + continue + } + if (!isReasonCode(listed.code)) { + results.push({ key, status: 'fail', reason: `"${listed.code}" is not one of the shared reason codes` }) + continue + } + results.push({ key, status: 'exempt', reason: `${listed.code}: ${listed.gate}` }) + } + + // An entry outliving the page that justified it. Nobody reads a list of + // exemptions looking for the one that no longer applies to anything. + for (const key of exempt.keys()) { + if (!Object.hasOwn(quoted, key)) { + results.push({ key, status: 'fail', reason: 'listed in NOT_IN_THE_ARENA but no page quotes it — delete the entry' }) + } + } + + return results +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const results = runCheck() + if (process.argv.includes('--json')) { + console.log(JSON.stringify(results, null, 2)) + } else { + for (const r of results) console.log(`${r.status.padEnd(7)} ${r.key} — ${r.reason}`) + } + const tally = (status) => results.filter((r) => r.status === status).length + console.log(`\nquote coverage: ${tally('run')} quoted source(s) run by the arena, ${tally('exempt')} exempt, ${tally('fail')} unaccounted for`) + if (tally('fail')) process.exitCode = 1 +} diff --git a/scripts/arena/check-quote-coverage.negative.mjs b/scripts/arena/check-quote-coverage.negative.mjs new file mode 100644 index 00000000..83a10b6f --- /dev/null +++ b/scripts/arena/check-quote-coverage.negative.mjs @@ -0,0 +1,126 @@ +#!/usr/bin/env node +/* + * Proof that check-quote-coverage.mjs fails, and fails for the stated reason, + * on each way a quoted example can go unaccounted for — and that the list of + * exemptions can only shrink. + * + * The controls matter as much as the defects: a check that rejected every + * input would satisfy all four failure cases and still be worthless, and this + * one has to keep passing the registry the repository actually ships. + * + * Reads nothing, so it runs wherever the rest of the suite does. + */ +import { NOT_IN_THE_ARENA, isReasonCode, runCheck } from './check-quote-coverage.mjs' + +let failures = 0 +function expect(name, results, key, wantStatus, fragment) { + const got = results.find((r) => r.key === key) + const ok = got && got.status === wantStatus && (!fragment || got.reason.includes(fragment)) + if (ok) console.log(`ok ${name}`) + else { + failures += 1 + console.error(`FAILED ${name}: ${got ? `got ${got.status} — ${got.reason}` : `no result for ${key}`}`) + } +} + +const exempt = (code) => new Map([['port:examples/excused.rs', { code, why: 'why', gate: 'the port runner' }]]) + +// Control: the registry this repository ships accounts for every quoted +// source. Without this the four defects below could all pass while the real +// check was broken. +const live = runCheck() +const unaccounted = live.filter((r) => r.status === 'fail') +if (unaccounted.length === 0) console.log('ok the shipped registry accounts for every quoted source') +else { + failures += 1 + console.error(`FAILED the shipped registry: ${unaccounted.map((r) => `${r.key} — ${r.reason}`).join('; ')}`) +} + +// Control: an artifact runs it, so nothing more is asked of it. +expect( + 'a quoted source an artifact runs', + runCheck({ quoted: { 'port:examples/run.rs': '' }, executed: ['port:examples/run.rs'], exempt: new Map() }), + 'port:examples/run.rs', + 'run', +) + +// Control: quoted, unrun, listed with a code from the table — the case the +// list exists for. +expect( + 'a quoted source listed with a reason code', + runCheck({ quoted: { 'port:examples/excused.rs': '' }, executed: [], exempt: exempt('serves-stdio') }), + 'port:examples/excused.rs', + 'exempt', + 'the port runner', +) + +// The defect this check was written for: a page quotes a program nothing in +// the arena runs, and nobody decided to allow it. +expect( + 'a quoted source nothing runs and nothing excuses', + runCheck({ quoted: { 'port:examples/new.rs': '' }, executed: [], exempt: new Map() }), + 'port:examples/new.rs', + 'fail', + 'run by no arena artifact', +) + +// The ratchet: an entry that an artifact now runs has to go, or the list +// becomes a record of things that used to be true. +expect( + 'an exemption an artifact now runs', + runCheck({ + quoted: { 'port:examples/excused.rs': '' }, + executed: ['port:examples/excused.rs'], + exempt: exempt('serves-stdio'), + }), + 'port:examples/excused.rs', + 'fail', + 'delete the entry', +) + +// The ratchet, the other way: the page went, so the exemption is excusing +// nothing. +expect( + 'an exemption no page quotes', + runCheck({ quoted: {}, executed: [], exempt: exempt('serves-stdio') }), + 'port:examples/excused.rs', + 'fail', + 'no page quotes it', +) + +// A reason has to cite the shared table. Free text here would let each +// exemption invent its own category, which is how a table stops meaning +// anything. +expect( + 'an exemption with an invented code', + runCheck({ quoted: { 'port:examples/excused.rs': '' }, executed: [], exempt: exempt('too-slow') }), + 'port:examples/excused.rs', + 'fail', + 'not one of the shared reason codes', +) + +// `platform:` carries a name, so it is matched by prefix rather than listed. +// Both halves are asserted: a bare `platform` is not a code. +for (const [code, want] of [['platform:windows', true], ['platform', false], ['platform:', false]]) { + if (isReasonCode(code) === want) console.log(`ok "${code}" ${want ? 'is' : 'is not'} a reason code`) + else { + failures += 1 + console.error(`FAILED "${code}": isReasonCode returned ${isReasonCode(code)}`) + } +} + +// Every shipped exemption cites the table, checked directly rather than only +// through the registry above, so a future entry cannot pass by being unquoted. +for (const [key, entry] of NOT_IN_THE_ARENA) { + if (isReasonCode(entry.code) && entry.gate && entry.why) console.log(`ok ${key} cites ${entry.code} and names its gate`) + else { + failures += 1 + console.error(`FAILED ${key}: code "${entry.code}", gate "${entry.gate}"`) + } +} + +if (failures) { + console.error(`quote coverage (negative): ${failures} expectation(s) failed`) + process.exit(1) +} +console.log('quote coverage (negative): every defect rejected; the controls passed') diff --git a/scripts/arena/check-quote-drift.mjs b/scripts/arena/check-quote-drift.mjs new file mode 100644 index 00000000..52dec937 --- /dev/null +++ b/scripts/arena/check-quote-drift.mjs @@ -0,0 +1,229 @@ +#!/usr/bin/env node +/* + * Compare what a page shows against what the arena runs. + * + * docs-arena.mjs counts a quoted source as run by matching a `slug:path` key, + * and a key says nothing about content: a port's `tmux-arena` worktree is free + * to diverge from the `-docs` worktree the site quotes, and nothing noticed. + * Two of them had, silently. + * + * Region-aware, because a file may legitimately differ outside the region a + * page quotes — arena evidence plumbing wrapped around a reader-facing body is + * exactly that — while the region itself must match byte for byte. A region + * missing from the run side is its own failure, never a quiet pass. + * + * Reads files and nothing else: no tmux, no toolchain, so it belongs in the + * gate every run executes rather than the port lane. + * + * Usage: + * node scripts/arena/check-quote-drift.mjs # check the worktrees + * node scripts/arena/check-quote-drift.mjs --json # machine-readable + */import { existsSync, readdirSync, readFileSync, statSync } from 'node:fs' +import { join, resolve } from 'node:path' +import { fileURLToPath } from 'node:url' +import sources from '../../site/src/data/example-sources.json' with { type: 'json' } +import { LANG_TO_PORT, checkoutFor, parseMeta, sliceRegion } from '../../site/src/plugins/remark-port-code.mjs' +import { ARTIFACTS, arenaWorktree } from './artifacts.mjs' + +const root = resolve(fileURLToPath(import.meta.url), '../../..') +const CONTENT = join(root, 'site/src/content/docs') + +function markdownFiles(dir) { + const out = [] + for (const name of readdirSync(dir)) { + const full = join(dir, name) + if (statSync(full).isDirectory()) out.push(...markdownFiles(full)) + else if (/\.mdx?$/.test(name)) out.push(full) + } + return out +} + +/** + * Every region a `file="..."` fence names for a given `slug:path` key, kept + * (unlike gen-example-sources.mjs's cache, which deliberately drops the + * region and caches whole files). A key with no region here is quoted whole. + * @returns {Map>} + */ +export function regionsByKey(contentDir = CONTENT) { + const out = new Map() + for (const md of markdownFiles(contentDir)) { + const text = readFileSync(md, 'utf8') + for (const m of text.matchAll(/^```(\w+)([^\n]*)$/gm)) { + const owner = LANG_TO_PORT[m[1]] + if (!owner) continue + const meta = parseMeta(m[2]) + if (!meta.file) continue + const key = `${owner}:${meta.file}` + if (!meta.region) continue + if (!out.has(key)) out.set(key, new Set()) + out.get(key).add(meta.region) + } + } + return out +} + +const norm = (text) => text.replace(/\s+$/, '') + +/** + * Whether `block` appears in `text` as consecutive lines, ignoring trailing + * whitespace and blank lines at either end. + * + * Used when the run side carries no marker to slice by: the question is then + * whether the lines a page shows are in the file, in order. + */ +export function containsLines(text, block) { + const lines = (source) => source.split('\n').map((line) => line.replace(/\s+$/, '')) + const wanted = lines(block).filter((line, index, all) => !(line === '' && (index === 0 || index === all.length - 1))) + const trimmed = wanted.filter((line, index) => !(line === '' && (index === 0 || index === wanted.length - 1))) + if (trimmed.length === 0) return true + const haystack = lines(text) + for (let start = 0; start + trimmed.length <= haystack.length; start += 1) { + let matched = true + for (let offset = 0; offset < trimmed.length; offset += 1) { + if (haystack[start + offset] !== trimmed[offset]) { + matched = false + break + } + } + if (matched) return true + } + return false +} + +/** + * Mismatches this repository has not closed yet, and why. + * + * A listed key reports `known` instead of failing. An unlisted mismatch fails, + * and a listed key that now matches fails too, so the list can only shrink — + * the same shape as the other ratchets here. Closing one means marking the + * region the page quotes on that port's `docs-site` branch, so the page shows + * the part the arena runs rather than a whole file that has grown plumbing. + */ +export const KNOWN_DRIFT = new Map([ + [ + 'java:examples/src/main/java/io/github/libtmux/examples/BuildAWorkspace.java', + 'the arena copy wraps the body the page quotes in evidence plumbing; needs a region marker', + ], + [ + 'dotnet:examples/LibTmux.Examples/Snippets/OneShot.cs', + 'the same shape: the page quotes the whole file, the arena copy adds its adapter', + ], +]) + +/** + * Fold a comparison into the record above. + * @param {{key: string, status: string, reason: string}} result + */ +export function applyKnownDrift(result) { + const known = KNOWN_DRIFT.get(result.key) + if (known === undefined) return result + if (result.status === 'fail') return { ...result, status: 'known', reason: known } + return { + ...result, + status: 'fail', + reason: `listed in KNOWN_DRIFT but it matches now — delete the entry: ${result.reason}`, + } +} + +/** + * Compare one quoted key against the file the arena runs. + * @param {string} key `slug:path` + * @param {string} quoted the cached/checkout content example-sources.json (or a live checkout) resolved + * @param {string} runPath absolute path to the file the arena executes + * @param {Set} [regions] + */ +export function compareOne(key, quoted, runPath, regions) { + if (!existsSync(runPath)) return { key, status: 'fail', reason: `the arena has no file at ${runPath}` } + const run = readFileSync(runPath, 'utf8') + + if (!regions || regions.size === 0) { + return norm(quoted) === norm(run) + ? { key, status: 'pass', reason: 'whole file matches' } + : { key, status: 'fail', reason: 'whole file content differs between the quoted source and the file the arena runs' } + } + + for (const region of regions) { + const quotedSlice = sliceRegion(quoted, region) + const runSlice = sliceRegion(run, region) + if (quotedSlice === null) return { key, status: 'fail', reason: `region "${region}" not found in the quoted source itself` } + if (runSlice === null) { + // Expected, not a defect: a region marker lives on the port's + // `docs-site` branch and never on the branch the arena runs from, so + // there is no marker here to slice by. What the page shows still has to + // be in the file, so compare the lines themselves. + if (!containsLines(run, quotedSlice)) { + return { key, status: 'fail', reason: `region "${region}" is not in the file the arena runs, with or without its markers` } + } + continue + } + if (norm(quotedSlice) !== norm(runSlice)) { + return { key, status: 'fail', reason: `region "${region}" content differs between the quoted source and the file the arena runs` } + } + } + return { key, status: 'pass', reason: `region(s) ${[...regions].join(', ')} match` } +} + +/** + * Resolve what a `slug:path` key currently shows a reader. + * + * "Quoted" means some page's `file="..."` fence actually resolved this key — + * recorded as a key in example-sources.json by gen-example-sources.mjs — not + * merely that a same-named file happens to exist in the port's checkout. + * rs:crates/libtmux/examples/inspect.rs and cxx:examples/01-tour.cpp are both + * real, readable files the arena runs, but no page fences them, so they must + * report as unquoted rather than being silently treated as quoted-and-run. + * Mirrors readFence's own precedence once a key is known to be quoted: the + * live checkout wins when present, the cache is the CI fallback. + */ +function resolveQuoted(key) { + if (!Object.hasOwn(sources, key)) return undefined + const [owner, ...rest] = key.split(':') + const path = rest.join(':') + const abs = join(checkoutFor(owner), path) + return existsSync(abs) ? readFileSync(abs, 'utf8') : sources[key] +} + +export function runCheck() { + const regions = regionsByKey() + const results = [] + for (const entry of ARTIFACTS) { + const worktree = arenaWorktree(entry.slug) + if (!existsSync(worktree)) { + // A port whose worktree is not here was not compared. Reporting it as + // drift would make an absent checkout look like a defect in the page. + results.push({ key: entry.slug, status: 'not run', reason: `no tmux-arena worktree at ${worktree}` }) + continue + } + for (const key of entry.runs) { + if (key.startsWith(`${entry.slug}:page:`)) { + // A page artifact is the executable unit itself, so no quoted-file + // entry can exist for it: python's doctest pages live in its own + // Sphinx tree, which gen-example-sources never scans. + results.push({ key, status: 'skip', reason: 'page artifact, not a quoted file — no example-sources.json entry can exist for it' }) + continue + } + const path = key.slice(entry.slug.length + 1) + const quoted = resolveQuoted(key) + if (quoted === undefined) { + results.push({ key, status: 'unquoted', reason: 'run by the arena but not quoted by any page' }) + continue + } + const runPath = join(worktree, path) + results.push(applyKnownDrift(compareOne(key, quoted, runPath, regions.get(key)))) + } + } + return results +} + +if (import.meta.url === `file://${process.argv[1]}`) { + const results = runCheck() + const asJson = process.argv.includes('--json') + if (asJson) { + console.log(JSON.stringify(results, null, 2)) + } else { + for (const r of results) console.log(`${r.status.padEnd(9)} ${r.key} — ${r.reason}`) + } + const failed = results.filter((r) => r.status === 'fail') + console.log(`\nquote-drift: ${results.length} run source(s) checked; ${failed.length} mismatch(es)`) + if (failed.length) process.exitCode = 1 +} diff --git a/scripts/arena/check-quote-drift.negative.mjs b/scripts/arena/check-quote-drift.negative.mjs new file mode 100644 index 00000000..c0e3496e --- /dev/null +++ b/scripts/arena/check-quote-drift.negative.mjs @@ -0,0 +1,144 @@ +#!/usr/bin/env node +/* + * Proof that check-quote-drift.mjs's region-aware comparison actually + * distinguishes "the file changed" from "the file changed but the quoted + * region didn't," and fails closed when a region can't be found at all. + * + * Reads files and nothing else, so it runs wherever the rest of the suite does. + */ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { KNOWN_DRIFT, applyKnownDrift, compareOne, containsLines } from './check-quote-drift.mjs' + +const dir = mkdtempSync(join(tmpdir(), 'quote-drift-negative-')) +const run = (name, content) => { + const path = join(dir, name) + writeFileSync(path, content) + return path +} + +let failures = 0 +function expect(name, result, wantStatus, fragment) { + const ok = result.status === wantStatus && (!fragment || result.reason.includes(fragment)) + if (ok) console.log(`ok ${name}`) + else { + failures += 1 + console.error(`FAILED ${name}: got ${result.status} — ${result.reason}`) + } +} + +// Control: identical whole file. +expect( + 'conforming whole file', + compareOne('k', 'line one\nline two\n', run('a.txt', 'line one\nline two\n')), + 'pass', +) + +// Whole-file drift: no region declared, content differs. +expect( + 'whole file drift', + compareOne('k', 'line one\nline two\n', run('b.txt', 'line one\nCHANGED\n')), + 'fail', + 'whole file content differs', +) + +// The java/dotnet shape: arena plumbing added around the reader-facing body. +// The whole file differs, but the marked region is byte-identical — must pass. +const quotedWithRegion = ['before', '// region: body', 'shared line', '// endregion', 'after'].join('\n') +const runWithSamePlumbingAdded = [ + 'before', + 'import extra.plumbing;', + '// region: body', + 'shared line', + '// endregion', + 'arena evidence printing', + 'after', +].join('\n') +expect( + 'region matches despite unrelated file drift', + compareOne('k', quotedWithRegion, run('c.txt', runWithSamePlumbingAdded), new Set(['body'])), + 'pass', +) + +// A marker lives on `docs-site` and the arena runs from another branch, so the +// run side has no marker to slice by. The lines still have to be there. +const runWithoutMarkersButSameLines = ['before', 'shared line', 'after'].join('\n') +expect( + 'region markers absent on the run side, lines present', + compareOne('k', quotedWithRegion, run('d-lines.txt', runWithoutMarkersButSameLines), new Set(['body'])), + 'pass', +) + +expect( + 'region markers absent on the run side, lines changed', + compareOne('k', quotedWithRegion, run('d-changed.txt', ['before', 'DIFFERENT line', 'after'].join('\n')), new Set(['body'])), + 'fail', + 'with or without its markers', +) + +// The swift Waiting.swift shape: the marker itself exists only on the quoted +// side; the arena's copy of the file never got it. +const runWithoutMarkerOrLines = ['before', 'after'].join('\n') +expect( + 'region missing on the run side entirely', + compareOne('k', quotedWithRegion, run('d.txt', runWithoutMarkerOrLines), new Set(['body'])), + 'fail', + 'with or without its markers', +) + +// The swift MCPEmbedding.swift shape: both sides carry the marker, but the +// text between the markers itself changed (a real API drift, not plumbing). +const runWithChangedRegion = ['before', '// region: body', 'DIFFERENT line', '// endregion', 'after'].join('\n') +expect( + 'region content differs on both sides', + compareOne('k', quotedWithRegion, run('e.txt', runWithChangedRegion), new Set(['body'])), + 'fail', + 'region "body" content differs', +) + +// Declaring a region the quoted source itself doesn't have is a fixture bug, +// not drift — must still fail, but distinguishably. +expect( + 'region missing on the quoted side', + compareOne('k', 'no markers here\n', run('f.txt', quotedWithRegion), new Set(['body'])), + 'fail', + 'not found in the quoted source itself', +) + +// The arena worktree lacks the file outright. +expect( + 'run file absent', + compareOne('k', 'anything', join(dir, 'does-not-exist.txt')), + 'fail', + 'has no file at', +) + +// The record of drift this repository has not closed yet only shrinks: a +// listed key that still mismatches is reported rather than failed, and one +// that has been fixed fails until its entry goes. +const listed = [...KNOWN_DRIFT.keys()][0] +expect( + 'a listed mismatch is reported, not failed', + applyKnownDrift({ key: listed, status: 'fail', reason: 'whole file content differs' }), + 'known', +) +expect( + 'a listed key that now matches fails until the entry goes', + applyKnownDrift({ key: listed, status: 'pass', reason: 'whole file matches' }), + 'fail', + 'delete the entry', +) +expect( + 'an unlisted mismatch still fails', + applyKnownDrift({ key: 'nobody:listed.txt', status: 'fail', reason: 'whole file content differs' }), + 'fail', +) + +rmSync(dir, { recursive: true, force: true }) + +if (failures) { + console.error(`check-quote-drift (negative): ${failures} expectation(s) failed`) + process.exit(1) +} +console.log('check-quote-drift (negative): every defect caught; every control passed') diff --git a/scripts/arena/fixture-adapter.mjs b/scripts/arena/fixture-adapter.mjs new file mode 100644 index 00000000..9b814b94 --- /dev/null +++ b/scripts/arena/fixture-adapter.mjs @@ -0,0 +1,72 @@ +#!/usr/bin/env node +/* + * A stand-in port adapter for `docs-arena.negative.mjs`. It follows the arena + * contract through the tmux CLI, and `DOCS_ARENA_FIXTURE_MODE` breaks exactly + * one part of it, so each supervisor check can be shown to fail. + */ +import { execFileSync, spawnSync } from 'node:child_process' + +const ARTIFACT = 'docs-arena-fixture' +const env = process.env +const mode = env.DOCS_ARENA_FIXTURE_MODE ?? 'conform' + +// Ordinary mode: without a descriptor there is nothing to borrow. +if (!env.LIBTMUX_ARENA_DESCRIPTOR) process.exit(0) + +const artifact = env.LIBTMUX_ARENA_ARTIFACT +const socket = env.LIBTMUX_SOCKET_PATH +const bin = env.LIBTMUX_TMUX_BIN +if (mode === 'ambient') spawnSync(bin || 'tmux', ['new-session', '-d', '-s', 'ambient'], { stdio: 'ignore' }) +if (!artifact || !socket || !bin || artifact !== ARTIFACT) { + if (mode === 'lenient') process.exit(0) + console.error('arena contract is incomplete or names another artifact') + process.exit(2) +} + +const tmux = (...args) => execFileSync(bin, ['-S', socket, ...args], { encoding: 'utf8' }).trim() +// The example body: real work on the borrowed server. +tmux('new-window', '-d', '-n', 'fixture') +if (mode === 'extra') spawnSync(bin, ['-L', 'extra', 'new-session', '-d'], { stdio: 'ignore' }) + +const [pid, reported] = tmux('display-message', '-p', '#{pid}\t#{socket_path}').split('\t') +const challenge = tmux('display-message', '-p', '#{@libtmux_arena_challenge}') +const base = { schema: 1, artifact, server_pid: Number(pid), socket_path: reported } +const record = JSON.stringify({ + ...base, + artifact: mode === 'artifact' ? 'another-artifact' : artifact, + challenge: mode === 'challenge' ? '0'.repeat(64) : challenge, + socket_path: mode === 'socket' ? `${reported}.elsewhere` : reported, +}) + +// N-records-per-run modes: one evidence line per declared source, reached +// only when the harness sets DOCS_ARENA_FIXTURE_SOURCES. Every mode above +// stays the single-record adapter docs-arena.negative.mjs exercises. +const sources = (env.DOCS_ARENA_FIXTURE_SOURCES ?? '').split(',').filter(Boolean) +if (sources.length) { + const print = (source) => console.log(`LIBTMUX_ARENA_EVIDENCE=${JSON.stringify({ ...base, challenge, source })}`) + const emit = mode === 'multi-missing' ? sources.slice(0, -1) : sources + for (const source of emit) { + if (mode === 'multi-replaced' && source === sources[1]) { + // What a documented block did to python's gate: stop the lent server, + // and let the next call quietly start another on the same socket. The + // records go on naming the pid and challenge the first server had. + tmux('kill-server') + spawnSync(bin, ['-S', socket, 'new-session', '-d'], { stdio: 'ignore' }) + } + if (mode === 'multi-bad-challenge' && source === sources[sources.length - 1]) { + console.log(`LIBTMUX_ARENA_EVIDENCE=${JSON.stringify({ ...base, challenge: '0'.repeat(64), source })}`) + } else { + print(source) + } + } + if (mode === 'multi-duplicate') print(sources[0]) + if (mode === 'multi-no-source') console.log(`LIBTMUX_ARENA_EVIDENCE=${JSON.stringify({ ...base, challenge })}`) + if (mode === 'multi-undeclared') print('an-undeclared-source') +} else if (mode === 'hang') { + setInterval(() => {}, 1000) +} else { + if (mode !== 'silent') console.log(`LIBTMUX_ARENA_EVIDENCE=${record}`) + if (mode === 'twice') console.log(`LIBTMUX_ARENA_EVIDENCE=${record}`) + if (mode === 'kill') tmux('kill-server') + if (mode === 'exit') process.exit(3) +} diff --git a/scripts/arena/supervisor.mjs b/scripts/arena/supervisor.mjs new file mode 100644 index 00000000..cf3e6af8 --- /dev/null +++ b/scripts/arena/supervisor.mjs @@ -0,0 +1,454 @@ +/* + * A tmux server this process owns, lent to one port example through that + * port's arena adapter. + * + * The contract is the one each port's `tmux-arena` branch implements. + * `LIBTMUX_ARENA_DESCRIPTOR` is the only activation signal; a complete + * contract adds the artifact name, the exact socket, and the exact tmux + * client. The adapter runs the example body through the port's production + * API, then prints one `LIBTMUX_ARENA_EVIDENCE=` line naming the live + * challenge, server PID, and socket it reached. It never stops the server. + * This module does, after proving the same server survived the run. + * + * `TMUX_TMPDIR` points into the private root for the server and the example, + * but it only routes default and `-L` sockets, so it is not what makes the + * server ours. Ownership comes from starting it here with `-D -S` and an + * empty config, and from authenticating its PID before anything is lent. A + * socket that appears under that `TMUX_TMPDIR` belongs to a server nobody was + * lent: leak evidence after a complete contract, and an ambient fallback + * after an incomplete one. + * + * Cleanup is process-group containment, not a process-tree reaper. A pane + * descendant that double-forks out of tmux's reach survives `kill-server`. + */ +import { spawn, spawnSync } from 'node:child_process' +import { randomBytes, timingSafeEqual } from 'node:crypto' +import { + accessSync, closeSync, constants, existsSync, lstatSync, mkdirSync, mkdtempSync, + openSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync, +} from 'node:fs' +import { tmpdir } from 'node:os' +import { delimiter, dirname, join } from 'node:path' +import { setTimeout as delay } from 'node:timers/promises' + +export const EVIDENCE_PREFIX = 'LIBTMUX_ARENA_EVIDENCE=' +export const CHALLENGE_OPTION = '@libtmux_arena_challenge' +// Short and fixed, because a Unix socket path has a small length limit. +// Where the lent server's private TMUX_TMPDIR is made. `TMPDIR` is honoured +// through tmpdir(), and LIBTMUX_DOCS_ARENA_ROOT names it outright, so a run +// that has been given a sandbox of its own cannot start a server outside it. +const ROOT_PARENT = process.env.LIBTMUX_DOCS_ARENA_ROOT ?? tmpdir() +const ROOT_PREFIX = 'lta-docs-' +const HOLD_SESSION = 'arena-hold' +const OUTPUT_LIMIT = 8 * 1024 * 1024 +const RECORD_LIMIT = 16 * 1024 +const TAIL = 2000 + +export class ArenaFailure extends Error {} + +const isExecutable = (path) => { + try { + accessSync(path, constants.X_OK) + return statSync(path).isFile() + } catch { + return false + } +} + +/** The tmux client every command here uses: an absolute override, or the first on PATH. */ +export function resolveTmux(override = process.env.LIBTMUX_DOCS_ARENA_TMUX) { + if (override) { + if (!override.startsWith('/') || !isExecutable(override)) throw new ArenaFailure(`no executable tmux at ${override}`) + return override + } + const found = (process.env.PATH ?? '').split(delimiter).filter(Boolean) + .map((dir) => join(dir, 'tmux')).find(isExecutable) + if (!found) throw new ArenaFailure('tmux is not on PATH') + return found +} + +function killGroup(pid, signal) { + if (!pid) return + try { + process.kill(-pid, signal) + } catch (error) { + if (error.code !== 'ESRCH') throw error + } +} + +const equal = (actual, expected) => typeof actual === 'string' && actual.length === expected.length + && timingSafeEqual(Buffer.from(actual), Buffer.from(expected)) +const tail = (text) => (text.length > TAIL ? `...${text.slice(-TAIL)}` : text) + +function tmux(arena, args, timeout = 2000) { + const result = spawnSync(arena.tmuxBin, ['-S', arena.socketPath, ...args], { + encoding: 'utf8', env: arena.serverEnv, timeout, + }) + return { ok: result.status === 0, out: (result.stdout ?? '').trim() } +} + +/** The PID of whatever answers at the lent socket, or undefined when nothing does. */ +function endpointPid(arena) { + try { + if (!lstatSync(arena.socketPath).isSocket()) return undefined + } catch { + return undefined + } + // display-message never starts a server, so a missing one stays missing. + const { ok, out } = tmux(arena, ['display-message', '-p', '#{pid}\t#{socket_path}'], 1000) + const [pid, socket] = out.split('\t') + return ok && socket === arena.socketPath && /^\d+$/.test(pid) ? Number(pid) : undefined +} + +/** Sockets under the private TMUX_TMPDIR: servers this run never lent. */ +function strays(arena) { + const dir = join(arena.runtime, `tmux-${process.getuid()}`) + return existsSync(dir) ? readdirSync(dir).map((name) => join(dir, name)) : [] +} + +function serverLog(arena) { + try { + return readFileSync(join(arena.root, 'server.log'), 'utf8').trim() + } catch { + return '' + } +} + +async function authenticate(arena, artifact) { + const started = Date.now() + let pid + while ((pid = endpointPid(arena)) === undefined) { + if (arena.spawnError || arena.server.exitCode !== null || arena.server.signalCode !== null) { + throw new ArenaFailure(`tmux exited before serving ${arena.socketPath}: ${arena.spawnError?.message ?? serverLog(arena)}`) + } + if (Date.now() - started > 5000) throw new ArenaFailure(`tmux did not serve ${arena.socketPath} in time`) + await delay(20) + } + if (pid !== arena.server.pid) { + throw new ArenaFailure(`the server at ${arena.socketPath} is PID ${pid}, not the one started here (${arena.server.pid})`) + } + if (!tmux(arena, ['new-session', '-d', '-s', HOLD_SESSION]).ok || endpointPid(arena) !== pid) { + throw new ArenaFailure('the tmux endpoint changed while it was being prepared') + } + const challenge = randomBytes(32).toString('hex') + if (!tmux(arena, ['set-option', '-gq', CHALLENGE_OPTION, challenge]).ok + || !equal(tmux(arena, ['show-options', '-gv', CHALLENGE_OPTION]).out, challenge)) { + throw new ArenaFailure('the tmux challenge could not be installed') + } + arena.pid = pid + arena.challenge = challenge + // The challenge stays out of the descriptor: an adapter has to read it from the live server. + arena.descriptor = join(arena.root, 'descriptor.json') + const descriptor = { + schema: 1, + artifact: { name: artifact }, + tmux: { executable: arena.tmuxBin, socket: arena.socketPath, pid, hold_session: HOLD_SESSION }, + } + writeFileSync(arena.descriptor, `${JSON.stringify(descriptor, null, 2)}\n`, { mode: 0o600 }) +} + +async function startArena(tmuxBin, artifact) { + const root = mkdtempSync(join(ROOT_PARENT, ROOT_PREFIX)) + const rootStat = lstatSync(root) + const dirs = Object.fromEntries(['home', 'config', 'runtime', 'tmp'].map((name) => [name, join(root, name)])) + for (const dir of Object.values(dirs)) mkdirSync(dir, { mode: 0o700 }) + const config = join(root, 'tmux.conf') + writeFileSync(config, '', { mode: 0o600 }) + // A private HOME with no startup file is a *new* account to an interactive + // shell, and zsh answers that with its first-run configuration wizard: the + // pane waits for a keypress and prints nothing. An example that sends keys + // and reads the pane back then fails on a timeout with empty output, which + // says nothing about the example. Four of them did. An empty startup file + // is what makes the account look configured; python's own fixture writes + // one for the same reason. + for (const startup of ['.zshrc', '.zshenv', '.bashrc', '.profile']) { + writeFileSync(join(dirs.home, startup), '', { mode: 0o600 }) + } + + // The example keeps HOME so its toolchain finds its caches; tmux never reads it, + // because the server was started with an explicit empty config. The server + // keeps the caller's SHELL, as an ordinary tmux would, but its panes load no + // dotfiles from a private HOME. Swapping in another shell changes how early + // keystrokes echo, and examples that match captured lines notice. + const inherited = Object.entries(process.env) + .filter(([key]) => key !== 'TMUX' && key !== 'TMUX_PANE' && !key.startsWith('LIBTMUX_')) + const artifactEnv = { + ...Object.fromEntries(inherited), + PATH: `${dirname(tmuxBin)}${delimiter}${process.env.PATH ?? ''}`, + TMUX_TMPDIR: dirs.runtime, + } + const serverEnv = { + ...artifactEnv, HOME: dirs.home, TMPDIR: dirs.tmp, + XDG_CONFIG_HOME: dirs.config, XDG_RUNTIME_DIR: dirs.runtime, + } + + const log = openSync(join(root, 'server.log'), 'w') + const socketPath = join(root, 's') + const server = spawn(tmuxBin, ['-D', '-S', socketPath, '-f', config], { + detached: true, env: serverEnv, stdio: ['ignore', log, log], + }) + closeSync(log) + const arena = { + root, rootIdentity: [rootStat.dev, rootStat.ino], runtime: dirs.runtime, + socketPath, tmuxBin, server, serverEnv, artifactEnv, + } + server.on('error', (error) => { + arena.spawnError = error + }) + try { + await authenticate(arena, artifact) + } catch (error) { + await stopArena(arena) + throw error + } + return arena +} + +function removeRoot({ root, rootIdentity: [dev, ino] }) { + let current + try { + current = lstatSync(root) + } catch { + return + } + if (!root.startsWith(join(ROOT_PARENT, ROOT_PREFIX)) || !current.isDirectory() + || current.dev !== dev || current.ino !== ino) { + throw new ArenaFailure(`refusing to remove ${root}: it is not the root this run created`) + } + rmSync(root, { recursive: true, force: true }) +} + +async function stopArena(arena) { + for (const socket of strays(arena)) { + spawnSync(arena.tmuxBin, ['-S', socket, 'kill-server'], { env: arena.serverEnv, stdio: 'ignore', timeout: 2000 }) + } + const { server } = arena + if (!arena.spawnError && server.exitCode === null && server.signalCode === null) { + const exited = new Promise((resolve) => server.once('exit', resolve)) + tmux(arena, ['kill-server']) + await Promise.race([exited, delay(2000)]) + } + killGroup(server.pid, 'SIGKILL') + removeRoot(arena) +} + +function runArtifact({ command, cwd, env, deadlineMs }) { + return new Promise((resolve) => { + const child = spawn(command[0], command.slice(1), { cwd, detached: true, env, stdio: ['ignore', 'pipe', 'pipe'] }) + const run = { stdout: '', stderr: '', overflow: false, timedOut: false } + const take = (key) => (chunk) => { + if (run[key].length + chunk.length > OUTPUT_LIMIT) run.overflow = true + else run[key] += chunk + } + child.stdout.setEncoding('utf8').on('data', take('stdout')) + child.stderr.setEncoding('utf8').on('data', take('stderr')) + const timer = setTimeout(() => { + run.timedOut = true + killGroup(child.pid, 'SIGKILL') + }, deadlineMs) + child.on('error', (error) => { + run.error = error + }) + child.on('close', (code, signal) => { + clearTimeout(timer) + // Nothing the example started in its own process group outlives it. + killGroup(child.pid, 'SIGKILL') + resolve({ ...run, code, signal }) + }) + }) +} + +function contractEnv(arena, artifact, extra, complete) { + return { + ...arena.artifactEnv, + ...extra, + LIBTMUX_ARENA_DESCRIPTOR: arena.descriptor, + LIBTMUX_ARENA_ARTIFACT: artifact, + LIBTMUX_SOCKET_PATH: complete ? arena.socketPath : '', + LIBTMUX_TMUX_BIN: arena.tmuxBin, + } +} + +const context = (run) => `\n--- stdout\n${tail(run.stdout)}\n--- stderr\n${tail(run.stderr)}` + +function requireFinished(run, deadlineMs, condition = '') { + if (run.error) throw new ArenaFailure(`the artifact did not start: ${run.error.message}`) + if (run.timedOut) throw new ArenaFailure(`the artifact missed its ${deadlineMs} ms deadline${condition}${context(run)}`) + if (run.overflow) throw new ArenaFailure(`the artifact printed more output than the arena keeps${context(run)}`) +} + +function requireSameServer(arena) { + if (arena.server.exitCode !== null || arena.server.signalCode !== null || endpointPid(arena) !== arena.pid) { + throw new ArenaFailure('the lent server exited or changed during the run') + } + if (!equal(tmux(arena, ['show-options', '-gv', CHALLENGE_OPTION]).out, arena.challenge)) { + throw new ArenaFailure('the lent server lost its challenge during the run') + } +} + +/** + * Every `LIBTMUX_ARENA_EVIDENCE=` line an adapter printed, parsed as + * JSON objects, in print order. No count is enforced here — callers that + * want exactly one record (parseEvidence) or exactly one per declared + * `source` (validateEvidenceBySource) enforce that themselves, so this stays + * the one place that knows how to find and parse a record line. + * + * python's gate already stamps a `source` field naming the page it ran, which + * nothing read. This is what makes it mean something: N records per run, one + * per declared source, beside the one-record-per-artifact contract runInArena + * still enforces unchanged. + */ +export function parseEvidenceRecords(stdout) { + return stdout.split('\n').filter((line) => line.startsWith(EVIDENCE_PREFIX)) + .map((line) => line.slice(EVIDENCE_PREFIX.length).replace(/\r$/, '')) + .map((line, index) => { + if (Buffer.byteLength(line) > RECORD_LIMIT) throw new ArenaFailure(`evidence record ${index} is larger than the arena accepts`) + let record + try { + record = JSON.parse(line) + } catch (error) { + throw new ArenaFailure(`evidence record ${index} is not JSON: ${error.message}`) + } + if (record === null || typeof record !== 'object' || Array.isArray(record)) { + throw new ArenaFailure(`evidence record ${index} is not a JSON object`) + } + return record + }) +} + +/** The one evidence record an adapter printed, parsed as a JSON object. */ +export function parseEvidence(stdout) { + const records = parseEvidenceRecords(stdout) + if (records.length !== 1) throw new ArenaFailure(`the adapter printed ${records.length} evidence records, not exactly one`) + return records[0] +} + +/** The checks every record must pass regardless of how many the adapter printed. */ +function validateOne(record, { artifact, pid, socketPath, challenge }) { + if (record.artifact !== artifact) { + throw new ArenaFailure(`the evidence names artifact ${JSON.stringify(record.artifact)}, not ${artifact}`) + } + if (record.schema !== 1 || !Number.isInteger(record.server_pid) || typeof record.socket_path !== 'string') { + throw new ArenaFailure('the evidence record has the wrong schema or field types') + } + if (typeof record.challenge !== 'string' || !/^[0-9a-f]{64}$/.test(record.challenge) || !equal(record.challenge, challenge)) { + throw new ArenaFailure('the evidence challenge is not the one the lent server holds') + } + if (record.socket_path !== socketPath) { + throw new ArenaFailure(`the evidence socket ${record.socket_path} is not the lent socket ${socketPath}`) + } + if (record.server_pid !== pid) throw new ArenaFailure(`the evidence PID ${record.server_pid} is not the lent server's ${pid}`) +} + +/** Prove the evidence names the lent artifact, socket, server, and live challenge. */ +export function validateEvidence(record, ctx) { + validateOne(record, ctx) +} + +/** + * Prove N evidence records — one per declared `source`, no more, no fewer — + * each pass every single-record check, and none collide or stray. + * + * Rejects, distinctly: a record missing `source`; a record naming a `source` + * this artifact never declared; two records naming the same `source`; and a + * declared `source` with no record at all. Order does not matter — sources + * are compared as a set, not a sequence, since an adapter may run them in + * any order. + */ +export function validateEvidenceBySource(records, { sources, artifact, pid, socketPath, challenge }) { + const declared = new Set(sources) + const seen = new Set() + for (const record of records) { + validateOne(record, { artifact, pid, socketPath, challenge }) + if (typeof record.source !== 'string' || record.source === '') { + throw new ArenaFailure('an evidence record is missing its source') + } + if (!declared.has(record.source)) { + throw new ArenaFailure(`the evidence names source ${JSON.stringify(record.source)}, which was not declared for this artifact`) + } + if (seen.has(record.source)) { + throw new ArenaFailure(`the evidence names source ${JSON.stringify(record.source)} more than once`) + } + seen.add(record.source) + } + const missing = sources.filter((source) => !seen.has(source)) + if (missing.length) { + throw new ArenaFailure(`no evidence record named declared source(s): ${missing.join(', ')}`) + } +} + +/** + * Lend one server to one run that is expected to print N evidence records — + * one per `sources` — instead of runInArena's exactly-one. Otherwise the + * identical contract: same server survival and stray-server checks, same + * fail-closed shape, just validated with validateEvidenceBySource instead of + * validateEvidence. + * + * One lend for several sources is what makes a doctest page affordable: five + * sources cost about 85 ms this way against about 421 ms as five lends. The + * server is still proven unchanged at the end, so a source that stops it and + * a later call that quietly starts another are caught, whichever source did + * it. + */ +export async function runInArenaMulti({ tmuxBin, artifact, sources, command, cwd, env = {}, deadlineMs = 180_000 }) { + const arena = await startArena(tmuxBin, artifact) + try { + const run = await runArtifact({ command, cwd, env: contractEnv(arena, artifact, env, true), deadlineMs }) + requireFinished(run, deadlineMs) + if (run.code !== 0) throw new ArenaFailure(`the artifact exited with ${run.code ?? run.signal}${context(run)}`) + const records = parseEvidenceRecords(run.stdout) + validateEvidenceBySource(records, { sources, artifact, pid: arena.pid, socketPath: arena.socketPath, challenge: arena.challenge }) + requireSameServer(arena) + const extra = strays(arena) + if (extra.length) throw new ArenaFailure(`an extra tmux server appeared under the private TMUX_TMPDIR: ${extra.join(', ')}`) + return records + } finally { + await stopArena(arena) + } +} + +/** + * Lend a fresh server to one artifact under the complete contract. Returns + * the evidence it printed once the evidence, the server's survival, and the + * absence of any other server have all been proven. + */ +export async function runInArena({ tmuxBin, artifact, command, cwd, env = {}, deadlineMs = 180_000 }) { + const arena = await startArena(tmuxBin, artifact) + try { + const run = await runArtifact({ command, cwd, env: contractEnv(arena, artifact, env, true), deadlineMs }) + requireFinished(run, deadlineMs) + if (run.code !== 0) throw new ArenaFailure(`the artifact exited with ${run.code ?? run.signal}${context(run)}`) + const evidence = parseEvidence(run.stdout) + validateEvidence(evidence, { artifact, pid: arena.pid, socketPath: arena.socketPath, challenge: arena.challenge }) + requireSameServer(arena) + const extra = strays(arena) + if (extra.length) throw new ArenaFailure(`an extra tmux server appeared under the private TMUX_TMPDIR: ${extra.join(', ')}`) + return evidence + } finally { + await stopArena(arena) + } +} + +/** + * Lend a fresh server under a contract whose socket is left out. The artifact + * has to fail, print no evidence, and reach no server at all: the adapter's + * fail-closed path, observed from outside. + */ +export async function runFailClosed({ tmuxBin, artifact, command, cwd, env = {}, deadlineMs = 180_000 }) { + const arena = await startArena(tmuxBin, artifact) + try { + const run = await runArtifact({ command, cwd, env: contractEnv(arena, artifact, env, false), deadlineMs }) + requireFinished(run, deadlineMs, ' under an incomplete contract') + const ambient = strays(arena) + if (ambient.length) { + throw new ArenaFailure(`under an incomplete contract the artifact reached for a default tmux server: ${ambient.join(', ')}`) + } + if (run.code === 0) throw new ArenaFailure(`the artifact exited 0 under an incomplete contract${context(run)}`) + if (run.stdout.split('\n').some((line) => line.startsWith(EVIDENCE_PREFIX))) { + throw new ArenaFailure('the artifact printed evidence under an incomplete contract') + } + requireSameServer(arena) + } finally { + await stopArena(arena) + } +} diff --git a/scripts/docs-arena.ev.negative.mjs b/scripts/docs-arena.ev.negative.mjs new file mode 100644 index 00000000..3463e663 --- /dev/null +++ b/scripts/docs-arena.ev.negative.mjs @@ -0,0 +1,69 @@ +#!/usr/bin/env node +/* + * Proof that runInArenaMulti rejects every way N evidence records can + * misreport which source produced which, and that a run whose server is + * replaced partway through is caught whichever source replaced it. + * + * Uses the same fixture adapter docs-arena.negative.mjs exercises for the + * one-record contract; DOCS_ARENA_FIXTURE_SOURCES switches it to N records. + * Starts and talks to a real tmux server, like that negative does. + */ +import { fileURLToPath } from 'node:url' +import { ArenaFailure, resolveTmux, runInArenaMulti } from './arena/supervisor.mjs' + +const fixture = fileURLToPath(new URL('./arena/fixture-adapter.mjs', import.meta.url)) +const tmuxBin = resolveTmux() +const SOURCES = ['docs/one.md', 'docs/two.md', 'docs/three.md'] +const target = (mode, sources = SOURCES, deadlineMs = 10_000) => ({ + tmuxBin, + artifact: 'docs-arena-fixture', + sources, + command: [process.execPath, fixture], + cwd: process.cwd(), + env: { DOCS_ARENA_FIXTURE_MODE: mode, DOCS_ARENA_FIXTURE_SOURCES: sources.join(',') }, + deadlineMs, +}) + +let failures = 0 +async function expectPass(name, run, wantCount) { + try { + const records = await run + if (wantCount !== undefined && records.length !== wantCount) { + failures += 1 + console.error(`FAILED ${name}: expected ${wantCount} record(s), got ${records.length}`) + } else { + console.log(`ok ${name} (${records.length} record(s))`) + } + } catch (error) { + failures += 1 + console.error(`FAILED ${name}: the control was rejected: ${error.message}`) + } +} +async function expectReject(name, run, fragment) { + try { + await run + failures += 1 + console.error(`FAILED ${name}: accepted`) + } catch (error) { + if (error instanceof ArenaFailure && error.message.includes(fragment)) console.log(`ok ${name} rejected`) + else { + failures += 1 + console.error(`FAILED ${name}: rejected for the wrong reason: ${error.message}`) + } + } +} + +await expectPass('conforming N-record adapter', runInArenaMulti(target('multi')), SOURCES.length) +await expectReject('duplicate source', runInArenaMulti(target('multi-duplicate')), 'more than once') +await expectReject('missing declared source', runInArenaMulti(target('multi-missing')), 'no evidence record named declared source') +await expectReject('a record missing its source field', runInArenaMulti(target('multi-no-source')), 'missing its source') +await expectReject('a mismatched challenge on one record', runInArenaMulti(target('multi-bad-challenge')), 'challenge is not the one') +await expectReject('a record for an undeclared source', runInArenaMulti(target('multi-undeclared')), 'not declared for this artifact') +await expectReject('the lent server replaced between sources', runInArenaMulti(target('multi-replaced')), 'exited or changed during the run') +await expectPass('a single declared source still works', runInArenaMulti(target('multi', ['docs/one.md'])), 1) + +if (failures) { + console.error(`docs arena (EV negative): ${failures} expectation(s) failed`) + process.exit(1) +} +console.log('docs arena (EV negative): every defect rejected; the control(s) passed') diff --git a/scripts/docs-arena.mjs b/scripts/docs-arena.mjs new file mode 100644 index 00000000..de73c2c5 --- /dev/null +++ b/scripts/docs-arena.mjs @@ -0,0 +1,131 @@ +#!/usr/bin/env node +/* + * Run the port examples these docs quote against a tmux server this script + * owns, through each port's arena adapter, then prove every adapter fails + * closed when its contract is incomplete. + * + * A port runs when its `tmux-arena` worktree and toolchain are present. One + * without them is reported as not run, never as passed; `--require` turns + * that into a failure. It starts with the quote coverage, which is a rule + * rather than a report: a page quoting a program no artifact runs fails here + * unless check-quote-coverage.mjs excuses it with a reason code. + * + * Usage: node scripts/docs-arena.mjs [--port ]... [--require] [--no-prepare] + */ +import { spawnSync } from 'node:child_process' +import { accessSync, constants, existsSync, mkdirSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { delimiter, join, resolve } from 'node:path' +import sources from '../site/src/data/example-sources.json' with { type: 'json' } +import { ARTIFACTS, arenaWorktree } from './arena/artifacts.mjs' +import { runCheck as checkQuoteCoverage } from './arena/check-quote-coverage.mjs' +import { ArenaFailure, resolveTmux, runFailClosed, runInArena, runInArenaMulti } from './arena/supervisor.mjs' + +const argv = process.argv.slice(2) +const only = new Set(argv.flatMap((arg, index) => (argv[index - 1] === '--port' ? [arg] : []))) +const requireAll = argv.includes('--require') +const prepare = !argv.includes('--no-prepare') + +const onPath = (tool) => (process.env.PATH ?? '').split(delimiter).filter(Boolean).some((dir) => { + try { + accessSync(join(dir, tool), constants.X_OK) + return true + } catch { + return false + } +}) + +function step(worktree, { cwd, command }) { + const result = spawnSync(command[0], command.slice(1), { + cwd: resolve(worktree, cwd), encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, stdio: ['ignore', 'pipe', 'pipe'], + }) + if (result.status !== 0) { + const output = `${result.stdout ?? ''}${result.stderr ?? ''}${result.error?.message ?? ''}`.trim() + throw new ArenaFailure(`preparing failed: ${command.join(' ')}\n${output.slice(-2000)}`) + } +} + +// Coverage has two kinds of key and they can never match each other. A quoted +// file is `slug:path`, recorded by gen-example-sources when a page fences it. +// A doctest page is `slug:page:path`: there the page is the executable unit, +// and it lives in the port's own documentation tree, which gen-example-sources +// never scans. Counted in one bucket, a page looked like a quoted file nothing +// quoted, and a file the arena runs but no page shows looked like coverage. +const isPage = (key) => key.split(':')[1] === 'page' +const executed = new Set(ARTIFACTS.flatMap((entry) => entry.runs)) +const executedFiles = [...executed].filter((key) => !isPage(key)) +const executedPages = [...executed].filter(isPage).sort() +// The quoted direction is a rule, not a report, and it lives in one place so +// that the every-run gate and this port lane cannot disagree about it: a page +// quoting a program the arena never runs fails here too. +const coverage = checkQuoteCoverage() +const covered = (status) => coverage.filter((result) => result.status === status).map((result) => result.key) +console.log(`quoted and run in the arena: ${covered('run').join(', ') || 'none'}`) +console.log(`quoted, not run here, exempt with a reason: ${covered('exempt').join(', ') || 'none'}`) +console.log(`run in the arena but quoted by no page: ${executedFiles.filter((key) => !Object.hasOwn(sources, key)).sort().join(', ') || 'none'}`) +console.log(`pages run in the arena: ${executedPages.join(', ') || 'none'}`) +for (const result of coverage.filter((entry) => entry.status === 'fail')) { + console.error(`quote coverage: ${result.key} — ${result.reason}`) +} + +let tmuxBin +let tmuxProblem +try { + tmuxBin = resolveTmux() +} catch (error) { + tmuxProblem = error.message +} + +const results = [] +const prepared = new Set() +for (const entry of ARTIFACTS) { + if (only.size && !only.has(entry.slug)) continue + const worktree = arenaWorktree(entry.slug) + const absent = tmuxProblem + ?? (existsSync(worktree) ? undefined : `no tmux-arena worktree at ${worktree}`) + ?? entry.tools.filter((tool) => !onPath(tool)).map((tool) => `${tool} is not on PATH`)[0] + if (absent) { + results.push({ slug: entry.slug, status: 'not run', detail: absent }) + continue + } + const build = join(tmpdir(), 'libtmux-docs-arena', entry.slug) + mkdirSync(build, { recursive: true }) + const startedAt = Date.now() + try { + if (prepare) { + for (const buildStep of entry.prepare(build)) { + // Artifacts of one port share install and build commands. Each + // distinct command runs once per worktree, so four ts examples do not + // pay for the same install and build four times. + const key = [worktree, buildStep.cwd, ...buildStep.command].join('\u0000') + if (prepared.has(key)) continue + step(worktree, buildStep) + prepared.add(key) + } + } + const { cwd, command } = entry.run(build) + const target = { tmuxBin, artifact: entry.artifact, command, cwd: resolve(worktree, cwd) } + // An artifact that declares several sources is lent one server for all of + // them and answers with one record each, so a page that produced none is + // caught. One source is still one record, which is what every other + // artifact does. + const evidence = entry.sources + ? (await runInArenaMulti({ ...target, sources: entry.sources }))[0] + : await runInArena(target) + await runFailClosed(target) + const ms = Date.now() - startedAt + results.push({ slug: entry.slug, status: 'pass', detail: `${entry.artifact} reached server ${evidence.server_pid}; failed closed without its socket (${ms}ms)` }) + } catch (error) { + if (!(error instanceof ArenaFailure)) throw error + const ms = Date.now() - startedAt + results.push({ slug: entry.slug, status: 'fail', detail: `${entry.artifact}: ${error.message} (${ms}ms)` }) + } +} + +for (const { slug, status, detail } of results) console.log(`${status.padEnd(7)} ${slug.padEnd(6)} ${detail}`) +const count = (status) => results.filter((result) => result.status === status).map((result) => result.slug) +const notRun = count('not run') +console.log(`docs arena: passed ${count('pass').join(', ') || 'none'}; failed ${count('fail').join(', ') || 'none'}; not run: ${notRun.join(', ') || 'none'}`) +const unaccounted = coverage.filter((result) => result.status === 'fail') +if (unaccounted.length) console.log(`docs arena: ${unaccounted.length} quoted source(s) unaccounted for, listed above`) +if (count('fail').length || unaccounted.length || (requireAll && notRun.length)) process.exitCode = 1 diff --git a/scripts/docs-arena.negative.mjs b/scripts/docs-arena.negative.mjs new file mode 100644 index 00000000..f5613ce6 --- /dev/null +++ b/scripts/docs-arena.negative.mjs @@ -0,0 +1,70 @@ +#!/usr/bin/env node +/* + * Proof that the docs arena rejects every way an adapter can claim contact it + * did not make, or reach a server it was not lent. + * + * The control is a fixture adapter that follows the contract, because a + * supervisor that rejected everything would satisfy each defect on its own. + * Each defect breaks one thing the supervisor checks: the evidence record's + * count, artifact, challenge, and socket; the lent server's survival; the + * exit status; the deadline; and whether any other server appears. Under an + * incomplete contract the control fails closed, and the defects exit 0 or + * fall back to a default server instead. + */ +import { fileURLToPath } from 'node:url' +import { ArenaFailure, resolveTmux, runFailClosed, runInArena } from './arena/supervisor.mjs' + +const fixture = fileURLToPath(new URL('./arena/fixture-adapter.mjs', import.meta.url)) +const tmuxBin = resolveTmux() +const target = (mode, deadlineMs = 10_000) => ({ + tmuxBin, + artifact: 'docs-arena-fixture', + command: [process.execPath, fixture], + cwd: process.cwd(), + env: { DOCS_ARENA_FIXTURE_MODE: mode }, + deadlineMs, +}) + +let failures = 0 +async function expectPass(name, run) { + try { + await run + console.log(`ok ${name}`) + } catch (error) { + failures += 1 + console.error(`FAILED ${name}: the control was rejected: ${error.message}`) + } +} +async function expectReject(name, run, fragment) { + try { + await run + failures += 1 + console.error(`FAILED ${name}: accepted`) + } catch (error) { + if (error instanceof ArenaFailure && error.message.includes(fragment)) console.log(`ok ${name} rejected`) + else { + failures += 1 + console.error(`FAILED ${name}: rejected for the wrong reason: ${error.message}`) + } + } +} + +await expectPass('conforming adapter', runInArena(target('conform'))) +await expectReject('no evidence', runInArena(target('silent')), 'evidence records') +await expectReject('two evidence records', runInArena(target('twice')), 'evidence records') +await expectReject('another artifact', runInArena(target('artifact')), 'names artifact') +await expectReject('a guessed challenge', runInArena(target('challenge')), 'challenge') +await expectReject('another socket', runInArena(target('socket')), 'lent socket') +await expectReject('adapter stops the lent server', runInArena(target('kill')), 'exited or changed') +await expectReject('nonzero exit after evidence', runInArena(target('exit')), 'exited with 3') +await expectReject('a missed deadline', runInArena(target('hang', 1_000)), 'deadline') +await expectReject('an extra server', runInArena(target('extra')), 'extra tmux server') +await expectPass('conforming adapter fails closed', runFailClosed(target('conform'))) +await expectReject('success under an incomplete contract', runFailClosed(target('lenient')), 'exited 0') +await expectReject('a default-server fallback', runFailClosed(target('ambient')), 'default tmux server') + +if (failures) { + console.error(`docs arena (negative): ${failures} expectation(s) failed`) + process.exit(1) +} +console.log('docs arena (negative): every defect rejected; the control passed') diff --git a/scripts/test-all.sh b/scripts/test-all.sh index bb4e52d3..a4812f64 100755 --- a/scripts/test-all.sh +++ b/scripts/test-all.sh @@ -106,6 +106,45 @@ node scripts/gen-example-sources.mjs --check step 'example sources (negative)' node scripts/gen-example-sources.negative.mjs +# The sources above are quoted; the docs arena runs them, each against a tmux +# server it owns and lends through that port's arena adapter. Its negative +# needs only tmux and proves the supervisor rejects every way an adapter can +# fake contact. The port lane needs each sibling tmux-arena worktree and its +# toolchain, so it runs only when asked for. +step 'docs arena (negative)' +node scripts/docs-arena.negative.mjs + +# One lend can serve several documented sources, which is what makes a doctest +# page affordable. This proves the supervisor still knows which source produced +# which record, and still notices a server replaced partway through. +step 'docs arena evidence (negative)' +node scripts/docs-arena.ev.negative.mjs + +# Running a source and quoting it are two different files until something +# compares them. This is that comparison; its negative needs no worktree and +# proves the comparison can fail, so it runs everywhere. +step 'quote drift (negative)' +node scripts/arena/check-quote-drift.negative.mjs + +# The other direction, and the one a new page gets wrong: fencing a program no +# arena artifact runs. It needs only the two data files, so unlike the drift +# comparison it belongs here rather than in the port lane — a page is written in +# a checkout that has no ports, which is exactly where it must fail. +step 'quote coverage' +node scripts/arena/check-quote-coverage.mjs + +step 'quote coverage (negative)' +node scripts/arena/check-quote-coverage.negative.mjs + +if [[ "${LIBTMUX_DOCS_ARENA:-}" == 1 ]]; then + step 'quote drift' + node scripts/arena/check-quote-drift.mjs + step 'docs arena' + node scripts/docs-arena.mjs --require +else + note_skip 'docs arena ports' +fi + step 'shell port table' node scripts/gen-shell-ports.mjs --check diff --git a/site/src/plugins/remark-port-code.mjs b/site/src/plugins/remark-port-code.mjs index 5b6ba87f..afdaf871 100644 --- a/site/src/plugins/remark-port-code.mjs +++ b/site/src/plugins/remark-port-code.mjs @@ -125,8 +125,12 @@ export function parseMeta(meta) { * Slice a file between `# region: name` / `# endregion` style markers, so a * page can quote one function out of a longer tested example. The comment * leader varies by language, so match the marker text rather than the syntax. + * + * Exported so the arena's quote-drift check slices the region out of the file + * it runs with the same matcher this plugin slices the page with. One + * implementation, so a region cannot mean two things depending on the caller. */ -function sliceRegion(source, region) { +export function sliceRegion(source, region) { const lines = source.split('\n') const start = lines.findIndex((l) => new RegExp(`region:\\s*${region}\\b`).test(l)) if (start === -1) return null