From 93ad8ea8cc79241681c02449399aceb710db1514 Mon Sep 17 00:00:00 2001 From: Brian Love Date: Sat, 29 Aug 2026 14:43:09 -0700 Subject: [PATCH] test(cockpit): repair the rotted manual specs and guard them in CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The *.manual.ts tier is unreachable: all 38 cockpit playwright configs use testMatch '**/*.spec.ts', and nothing in any config, project target, or workflow references it. 34 live-LLM specs that nobody runs and CI cannot see. Audited all 34. Ports were fine (0 drift vs cockpit/ports.mjs) and no vacuous neg-only assertions remained. But 10 asserted UI copy that exists NOWHERE in the repo: - deep-agents/{filesystem,memory,planning,sandboxes,skills,subagents}: welcome-prompt strings replaced by empty-state copy ('No delegations yet', 'No plan yet', ...) - langgraph/durable-execution: 'Execution Status' -> 'Pipeline' - langgraph/memory: 'Agent Memory' -> 'Learned Facts' - {langgraph,ag-ui}/interrupts: asserted an approvals sidebar that no longer exists — those examples moved to a chat-approval-card modal with welcome suggestions. Now assert the suggestion, and the test name no longer describes a sidebar. Repairing them isn't enough — nothing would stop the next drift. Since the specs can't run in CI (real model, per-example dev server), this adds a tripwire that can: apps/cockpit/scripts/manual-spec-freshness.spec.ts checks every text and element selector a manual spec asserts still exists in the source. It lives in apps/cockpit because NX It's time to update Nx 🎉 Your repository uses a higher version of Nx (22.5.1) than your global CLI version (21.5.2) For more information, see https://nx.dev/more-concepts/global-nx > nx run cockpit:test NX Successfully ran target test for project cockpit actually runs in CI — libs/e2e-harness would have reproduced the original problem, since it isn't in the Library job's LIBS list. 69 assertions (34 specs x 2 checks + a self-guard that fails if the glob ever stops matching the tier). Mutation-tested: reintroducing the exact stale string this commit removes turns it red and names the file. Co-Authored-By: Claude Opus 5 --- .../scripts/manual-spec-freshness.spec.ts | 93 +++++++++++++++++++ .../angular/e2e/manual/interrupts.manual.ts | 4 +- .../angular/e2e/manual/filesystem.manual.ts | 2 +- .../angular/e2e/manual/memory.manual.ts | 2 +- .../angular/e2e/manual/planning.manual.ts | 2 +- .../angular/e2e/manual/sandboxes.manual.ts | 2 +- .../angular/e2e/manual/skills.manual.ts | 2 +- .../angular/e2e/manual/subagents.manual.ts | 2 +- .../e2e/manual/durable-execution.manual.ts | 2 +- .../angular/e2e/manual/interrupts.manual.ts | 4 +- .../angular/e2e/manual/memory.manual.ts | 2 +- 11 files changed, 105 insertions(+), 12 deletions(-) create mode 100644 apps/cockpit/scripts/manual-spec-freshness.spec.ts diff --git a/apps/cockpit/scripts/manual-spec-freshness.spec.ts b/apps/cockpit/scripts/manual-spec-freshness.spec.ts new file mode 100644 index 000000000..df647cdec --- /dev/null +++ b/apps/cockpit/scripts/manual-spec-freshness.spec.ts @@ -0,0 +1,93 @@ +// SPDX-License-Identifier: MIT +// +// Tripwire for the `*.manual.ts` tier. +// +// Manual specs are live-LLM checks a human runs by hand: every cockpit +// playwright config matches `**/*.spec.ts`, so nothing executes these and CI +// can never notice when they drift. When this guard was written, 10 of 34 were +// asserting UI copy that existed nowhere in the repo — panel headings that had +// been renamed, and empty-state strings from sidebars that no longer exist. +// +// Running the specs themselves in CI isn't viable (real model, per-example dev +// server). What IS cheap is checking that everything they assert still exists +// in the source. That catches the rot this tier actually suffers — stale copy +// and dropped selectors — without booting anything. +import { readFileSync, readdirSync, statSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { describe, expect, it } from 'vitest'; + +const REPO_ROOT = resolve(__dirname, '../../..'); +const SKIP_DIRS = new Set(['node_modules', 'dist', '.angular', '.venv', '.next', 'coverage']); + +function walk(dir: string, out: string[] = []): string[] { + let entries; + try { + entries = readdirSync(dir, { withFileTypes: true }); + } catch { + return out; + } + for (const entry of entries) { + if (entry.isDirectory()) { + if (!SKIP_DIRS.has(entry.name)) walk(join(dir, entry.name), out); + } else { + out.push(join(dir, entry.name)); + } + } + return out; +} + +const allFiles = walk(join(REPO_ROOT, 'cockpit')).concat(walk(join(REPO_ROOT, 'libs'))); +const manualSpecs = allFiles.filter((f) => f.endsWith('.manual.ts')); + +/** Every source file a manual spec could legitimately be asserting against. */ +const sourceCorpus = allFiles + .filter((f) => /\.(ts|html|css)$/.test(f) && !f.endsWith('.manual.ts')) + .map((f) => { + try { + return readFileSync(f, 'utf8'); + } catch { + return ''; + } + }) + .join('\n'); + +/** `text=Some copy` and `getByText('Some copy')` assertions. */ +function assertedText(spec: string): string[] { + const out = new Set(); + for (const m of spec.matchAll(/text=([^'"`)]{4,80})/g)) out.add(m[1].trim()); + for (const m of spec.matchAll(/getByText\(\s*['"]([^'"]{4,80})['"]/g)) out.add(m[1].trim()); + return [...out]; +} + +/** `page.locator('some-element')` custom-element selectors. */ +function assertedSelectors(spec: string): string[] { + const out = new Set(); + for (const m of spec.matchAll(/locator\(\s*['"]([a-z][a-z0-9]*(?:-[a-z0-9]+)+)['"]/g)) out.add(m[1]); + return [...out]; +} + +describe('manual e2e specs stay in sync with the source', () => { + it('finds the manual tier', () => { + // Guards the guard: if the glob ever stops matching, every assertion below + // would pass over an empty list and this file would be worthless. + expect(manualSpecs.length).toBeGreaterThan(20); + }); + + it.each(manualSpecs.map((f) => [f.slice(REPO_ROOT.length + 1), f]))( + '%s asserts only text that still exists', + (_label, file) => { + const spec = readFileSync(file, 'utf8'); + const missing = assertedText(spec).filter((t) => !sourceCorpus.includes(t)); + expect(missing, `copy asserted by this manual spec no longer exists anywhere in the repo`).toEqual([]); + }, + ); + + it.each(manualSpecs.map((f) => [f.slice(REPO_ROOT.length + 1), f]))( + '%s targets only selectors that still exist', + (_label, file) => { + const spec = readFileSync(file, 'utf8'); + const missing = assertedSelectors(spec).filter((s) => !sourceCorpus.includes(s)); + expect(missing, `element selectors targeted by this manual spec no longer exist`).toEqual([]); + }, + ); +}); diff --git a/cockpit/ag-ui/interrupts/angular/e2e/manual/interrupts.manual.ts b/cockpit/ag-ui/interrupts/angular/e2e/manual/interrupts.manual.ts index 6bbc1bad8..2e997ed6e 100644 --- a/cockpit/ag-ui/interrupts/angular/e2e/manual/interrupts.manual.ts +++ b/cockpit/ag-ui/interrupts/angular/e2e/manual/interrupts.manual.ts @@ -18,10 +18,10 @@ test.describe('AG-UI Interrupts Example', () => { await page.waitForSelector('app-interrupts', { state: 'attached' }); }); - test('renders the chat interface with approvals sidebar', async ({ page }) => { + test('renders the chat interface with refund suggestions', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=No pending approvals')).toBeVisible(); + await expect(page.locator('text=Refund a duplicate charge')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/filesystem/angular/e2e/manual/filesystem.manual.ts b/cockpit/deep-agents/filesystem/angular/e2e/manual/filesystem.manual.ts index b300259c0..6d192e231 100644 --- a/cockpit/deep-agents/filesystem/angular/e2e/manual/filesystem.manual.ts +++ b/cockpit/deep-agents/filesystem/angular/e2e/manual/filesystem.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Filesystem Example', () => { test('renders the chat interface with file operations sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask the agent to read or write a file.')).toBeVisible(); + await expect(page.locator('text=No file operations yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/memory/angular/e2e/manual/memory.manual.ts b/cockpit/deep-agents/memory/angular/e2e/manual/memory.manual.ts index 8e2f6d1a2..21d64cec6 100644 --- a/cockpit/deep-agents/memory/angular/e2e/manual/memory.manual.ts +++ b/cockpit/deep-agents/memory/angular/e2e/manual/memory.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Memory Example', () => { test('renders the chat interface with memory sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Tell the agent something about yourself to see it remember.')).toBeVisible(); + await expect(page.locator('text=No facts learned yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/planning/angular/e2e/manual/planning.manual.ts b/cockpit/deep-agents/planning/angular/e2e/manual/planning.manual.ts index 608bd516f..e90e4bdc9 100644 --- a/cockpit/deep-agents/planning/angular/e2e/manual/planning.manual.ts +++ b/cockpit/deep-agents/planning/angular/e2e/manual/planning.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Planning Example', () => { test('renders the chat interface with plan sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask a complex question to see the plan.')).toBeVisible(); + await expect(page.locator('text=No plan yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/sandboxes/angular/e2e/manual/sandboxes.manual.ts b/cockpit/deep-agents/sandboxes/angular/e2e/manual/sandboxes.manual.ts index a0a70b7ec..f177b6e36 100644 --- a/cockpit/deep-agents/sandboxes/angular/e2e/manual/sandboxes.manual.ts +++ b/cockpit/deep-agents/sandboxes/angular/e2e/manual/sandboxes.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Sandboxes Example', () => { test('renders the chat interface with execution log sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask the agent to write and run Python code.')).toBeVisible(); + await expect(page.locator('text=No code executed yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/skills/angular/e2e/manual/skills.manual.ts b/cockpit/deep-agents/skills/angular/e2e/manual/skills.manual.ts index 48fe4f979..05fd2b405 100644 --- a/cockpit/deep-agents/skills/angular/e2e/manual/skills.manual.ts +++ b/cockpit/deep-agents/skills/angular/e2e/manual/skills.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Skills Example', () => { test('renders the chat interface with skill invocation sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask the agent to calculate, count words, or summarize text.')).toBeVisible(); + await expect(page.locator('text=No skills invoked yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/deep-agents/subagents/angular/e2e/manual/subagents.manual.ts b/cockpit/deep-agents/subagents/angular/e2e/manual/subagents.manual.ts index 772857d66..7cfa4d1e2 100644 --- a/cockpit/deep-agents/subagents/angular/e2e/manual/subagents.manual.ts +++ b/cockpit/deep-agents/subagents/angular/e2e/manual/subagents.manual.ts @@ -9,7 +9,7 @@ test.describe('Deep Agents Subagents Example', () => { test('renders the chat interface with subagents sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Ask a question to see subagent activity.')).toBeVisible(); + await expect(page.locator('text=No delegations yet')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/langgraph/durable-execution/angular/e2e/manual/durable-execution.manual.ts b/cockpit/langgraph/durable-execution/angular/e2e/manual/durable-execution.manual.ts index 5c943f59e..e8b2cd8ec 100644 --- a/cockpit/langgraph/durable-execution/angular/e2e/manual/durable-execution.manual.ts +++ b/cockpit/langgraph/durable-execution/angular/e2e/manual/durable-execution.manual.ts @@ -9,7 +9,7 @@ test.describe('LangGraph Durable Execution Example', () => { test('renders the chat interface with execution status sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Execution Status')).toBeVisible(); + await expect(page.locator('text=Pipeline')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/langgraph/interrupts/angular/e2e/manual/interrupts.manual.ts b/cockpit/langgraph/interrupts/angular/e2e/manual/interrupts.manual.ts index 7025339c5..847728aa6 100644 --- a/cockpit/langgraph/interrupts/angular/e2e/manual/interrupts.manual.ts +++ b/cockpit/langgraph/interrupts/angular/e2e/manual/interrupts.manual.ts @@ -6,10 +6,10 @@ test.describe('LangGraph Interrupts Example', () => { await page.waitForSelector('app-interrupts', { state: 'attached' }); }); - test('renders the chat interface with approvals sidebar', async ({ page }) => { + test('renders the chat interface with refund suggestions', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=No pending approvals')).toBeVisible(); + await expect(page.locator('text=Refund a duplicate charge')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => { diff --git a/cockpit/langgraph/memory/angular/e2e/manual/memory.manual.ts b/cockpit/langgraph/memory/angular/e2e/manual/memory.manual.ts index 6b76ca5da..5222c4b99 100644 --- a/cockpit/langgraph/memory/angular/e2e/manual/memory.manual.ts +++ b/cockpit/langgraph/memory/angular/e2e/manual/memory.manual.ts @@ -9,7 +9,7 @@ test.describe('LangGraph Memory Example', () => { test('renders the chat interface with memory sidebar', async ({ page }) => { await expect(page.locator('chat')).toBeVisible(); await expect(page.locator('textarea[name="messageText"]')).toBeVisible(); - await expect(page.locator('text=Agent Memory')).toBeVisible(); + await expect(page.locator('text=Learned Facts')).toBeVisible(); }); test('sends a message and receives a response', async ({ page }) => {