diff --git a/grammars/tool-call.gbnf b/grammars/tool-call.gbnf index d4ac9225..c749222e 100644 --- a/grammars/tool-call.gbnf +++ b/grammars/tool-call.gbnf @@ -5,7 +5,7 @@ root ::= tool-call-array tool-call ::= "{" ws "\"tool\"" ws ":" ws tool-name ws "," ws "\"args\"" ws ":" ws object ws "}" tool-call-array ::= "[" ws tool-call ( ws "," ws tool-call ){0,15} ws "]" -tool-name ::= browser-tool | os-tool | discovery-tool | memory-tool | tasks-tool | vision-tool | fusion-tool | mcp-native-tool | mcp-server-tool | "\"reply\"" | "\"finish\"" +tool-name ::= browser-tool | os-tool | discovery-tool | memory-tool | tasks-tool | vision-tool | fusion-tool | verify-tool | mcp-native-tool | mcp-server-tool | "\"reply\"" | "\"finish\"" browser-tool ::= "\"browser." ( "navigate" | "click" | "type" | "read_aria" | "search" | "tabs" | "scroll" ) "\"" os-tool ::= "\"os." ( "shell.run" | "fs.read" | "fs.read_document" | "fs.write" | "fs.trash" | "fs.list" | "fs.grep" | "fs.glob" | "fs.locate_project" | "fs.edit" | "fs.hash" | "fs.diff" | "fs.patch" | "fs.watch" | "fs.archive.list" | "fs.archive.read_entry" | "fs.archive.extract" | "http.request" | "web.search" | "web.fetch" | "git.status" | "git.log" | "git.diff" | "git.show" | "git.blame" | "git.branch" | "git.init" | "git.add" | "git.commit" | "git.checkout" | "git.clone" | "git.remote" | "git.fetch" | "git.pull" | "git.push" | "proc.list" | "proc.kill" | "clipboard.read" | "clipboard.write" | "window.list" | "window.focus" | "notify" | "email.inbox" | "email.send" ) "\"" discovery-tool ::= "\"skill." ( "view" | "run_script" ) "\"" | "\"tool.view\"" @@ -23,6 +23,10 @@ vision-tool ::= "\"vision.describe\"" # also grammar-constrained) is refused by `fusion.delegate` itself when # it tries to fan out again. fusion-tool ::= "\"fusion.delegate\"" +# Read-only verification (syntax per file; a command, service or page +# run against a throwaway copy). A local orchestrator reviewing a fan-out +# needs these in its vocabulary for the same reason it needs the fan-out. +verify-tool ::= "\"verify." ( "syntax" | "run" ) "\"" # Native MCP discovery / resource / prompt tools — always available. mcp-native-tool ::= "\"mcp." ( "resource.list" | "resource.read" | "prompt.list" | "prompt.get" ) "\"" # Per-server MCP tool branch. The static fallback (`mcp..`) diff --git a/src/agent/fusion-orchestrator-mode.test.ts b/src/agent/fusion-orchestrator-mode.test.ts index 59dd6bd0..0080e4d8 100644 --- a/src/agent/fusion-orchestrator-mode.test.ts +++ b/src/agent/fusion-orchestrator-mode.test.ts @@ -1,4 +1,6 @@ import { describe, expect, it } from "vitest"; +import type { ApprovalGate } from "../approval/approval-gate.js"; +import { buildVerifyRunTool, verifySyntaxTool } from "../tools/verify/index.js"; import { checkFusionOrchestrator, emptyFusionOrchestratorState, @@ -8,6 +10,23 @@ import { wouldRefuse, } from "./fusion-orchestrator-mode.js"; +/** The flag as the shipped definition carries it; the gate never runs the tool here. */ +const verifyRunReadonly = buildVerifyRunTool({ + approvals: {} as ApprovalGate, + approvalRequired: false, + config: { + browser: { + enabled: false, + channel: "chrome", + headless: true, + cdpUrl: null, + executablePath: null, + noSandbox: false, + launchTimeoutMs: 1_000, + }, + }, +}).readonly; + /** The two facts the gate reads off a tool: does it exist, does it mutate. */ function registryWith( tools: Record, @@ -67,6 +86,19 @@ describe("the fusion orchestrator gate", () => { ).toBe(true); }); + it("lets the orchestrator verify — read-only checks are review, not building (D1)", () => { + // The real definitions, not a fixture: the gate reads `readonly` + // off the registry, so this pins the flag the tools actually ship. + const registry = registryWith({ + "verify.syntax": { readonly: verifySyntaxTool.readonly }, + "verify.run": { readonly: verifyRunReadonly }, + }); + for (const tool of ["verify.syntax", "verify.run"]) { + expect(checkFusionOrchestrator(tool, registry, BEFORE).allowed).toBe(true); + expect(checkFusionOrchestrator(tool, registry, AFTER).allowed).toBe(true); + } + }); + it("never gates the terminal verbs", () => { // Vetoing `reply` would veto the turn's own exit — the mistake // plan mode documents and avoids for the same reason. diff --git a/src/agent/tool-resource-class.ts b/src/agent/tool-resource-class.ts index 5c36fd84..bba5d4e6 100644 --- a/src/agent/tool-resource-class.ts +++ b/src/agent/tool-resource-class.ts @@ -178,6 +178,15 @@ const TOOL_RESOURCE_CLASS: Record = { // stack the orchestrator's own fan-out on top of the fan-out. "fusion.delegate": "approval_gated", + // verify.* — read-only checks. `verify.syntax` spawns nothing that + // writes (node --check, tsc --noEmit, python compile(), bash -n), so + // several may run side by side. + "verify.syntax": "pure_read", + // `verify.run` executes a command, a server or a page — in a + // throwaway copy, so the workspace is untouched, but it still asks + // under the shell category below level 4, and it must be solo. + "verify.run": "approval_gated", + // mcp.* discovery / read tools — pure_read regardless of per-server // trust because they only inspect the local catalog or fetch // declared resources/prompts. Per-server-tool calls go through @@ -283,6 +292,8 @@ const FS_WRITE_CATEGORIES: readonly ApprovalCategory[] = [ const APPROVAL_CATEGORIES_BY_TOOL: Record = { "os.shell.run": ["shell"], + // Same rung as a shell command: it runs one, in a copy. + "verify.run": ["shell"], "os.fs.write": FS_WRITE_CATEGORIES, "os.fs.edit": FS_WRITE_CATEGORIES, "os.fs.patch": FS_WRITE_CATEGORIES, diff --git a/src/llm/grammar/build-grammar.test.ts b/src/llm/grammar/build-grammar.test.ts index 4ae0b121..d2ff990a 100644 --- a/src/llm/grammar/build-grammar.test.ts +++ b/src/llm/grammar/build-grammar.test.ts @@ -140,6 +140,25 @@ describe("fusion.delegate in the local-model grammar", () => { }); }); +describe("verify.* in the local-model grammar", () => { + it("admits both verification tools, so a local orchestrator can review a fan-out", async () => { + for (const profile of [ + PLAIN_INSTRUCT_PROFILE, + QWEN_THINK_PROFILE, + GEMMA4_THINK_PROFILE, + ]) { + const grammar = await buildGrammar(profile); + const toolName = + grammar.split("\n").find((l) => l.startsWith("tool-name ::=")) ?? ""; + expect(toolName, profile.id).toContain("verify-tool"); + const rule = + grammar.split("\n").find((l) => l.startsWith("verify-tool ::=")) ?? ""; + expect(rule, profile.id).toContain('"syntax"'); + expect(rule, profile.id).toContain('"run"'); + } + }); +}); + describe("os-tool names the local-model grammar admits", () => { it("includes the agent's e-mail tools — a descriptor the grammar cannot emit is a tool local models cannot call", () => { const { readFileSync } = require("node:fs") as typeof import("node:fs"); diff --git a/src/llm/provider/openai/openai-strict-tools.test.ts b/src/llm/provider/openai/openai-strict-tools.test.ts index 51ce5934..4c1980eb 100644 --- a/src/llm/provider/openai/openai-strict-tools.test.ts +++ b/src/llm/provider/openai/openai-strict-tools.test.ts @@ -483,15 +483,17 @@ describe("strict-tool conformance over every registered tool", () => { .map((tool) => String((tool.function as Schema).name)); // `os.http.request` carries a free-form header map and a free-form // JSON body; `mcp.prompt.get` forwards a server-defined argument - // map. Both are the tool's actual payload, so neither can be closed. + // map; `verify.run` takes an `env` map of arbitrary variable names. // `fusion.delegate` joined them with its `contract`: `owners` is a // path → task map, and each `checks` entry is a `verify.run` spec // whose keys belong to that tool — closing it would leave the model // unable to write a check at all. `parseDelegateArgs` validates the - // shape at run time, as it always did. + // shape at run time, as it always did. Each is the tool's actual + // payload, so none can be closed. expect(refused).toEqual([ "os__http__request", "mcp__prompt__get", + "verify__run", "fusion__delegate", ]); }); diff --git a/src/llm/provider/openai/strict-tool-schema.test.ts b/src/llm/provider/openai/strict-tool-schema.test.ts index 92114d88..eb3f79d9 100644 --- a/src/llm/provider/openai/strict-tool-schema.test.ts +++ b/src/llm/provider/openai/strict-tool-schema.test.ts @@ -532,7 +532,7 @@ describe("toStrictJsonSchema", () => { * knowingly — either the schema loses a bound it does not need, or * the tool joins this list. */ - it("converts all but five of the sampled built-in tool schemas", () => { + it("converts all but six of the sampled built-in tool schemas", () => { const refused: string[] = []; let converted = 0; for (const name of DEFAULT_TOOL_NAMES) { @@ -550,14 +550,16 @@ describe("toStrictJsonSchema", () => { "os.fs.archive.extract", // `headers` is a map; `body` may be any object. "os.http.request", + // `env` is a map of arbitrary variable names. + "verify.run", // maxItems on `paths`. "vision.describe", ]); expect(converted).toBe(DEFAULT_TOOL_NAMES.length - refused.length); - // 82 registered schemas, 77 of them strict. Pinned as a number so + // 84 registered schemas, 78 of them strict. Pinned as a number so // the sample cannot quietly shrink. - expect(DEFAULT_TOOL_NAMES.length).toBe(82); - expect(converted).toBe(77); + expect(DEFAULT_TOOL_NAMES.length).toBe(84); + expect(converted).toBe(78); }); /** @@ -671,6 +673,8 @@ const DEFAULT_TOOL_NAMES: readonly string[] = [ "mcp.prompt.list", "mcp.prompt.get", "fusion.delegate", + "verify.syntax", + "verify.run", // The nine from `github-tool-args-schemas.ts`, spread into the same // registry. Left out of this list, a bound added to one of them would // have joined the refusal set silently — the exact surprise the pin diff --git a/src/prompt/default-tool-args-schemas.ts b/src/prompt/default-tool-args-schemas.ts index 570942e4..7806a0bf 100644 --- a/src/prompt/default-tool-args-schemas.ts +++ b/src/prompt/default-tool-args-schemas.ts @@ -769,6 +769,58 @@ const DEFAULT_TOOL_ARGS_SCHEMAS: ReadonlyMap = new Map< ["tasks"], ), ], + ["verify.syntax", obj({ files: stringArraySchema }, ["files"])], + [ + "verify.run", + obj( + { + kind: { type: "string", enum: ["command", "service", "page"] }, + cwd: stringSchema, + // A map of arbitrary keys, like `os.http.request.headers`; the + // strict converter refuses it, knowingly. + env: { type: "object", additionalProperties: { type: "string" } }, + timeoutMs: numberSchema, + network: booleanSchema, + cmd: stringSchema, + args: stringArraySchema, + start: obj({ cmd: stringSchema, args: stringArraySchema }, ["cmd"]), + ready: obj({ port: integerSchema, url: stringSchema, timeoutMs: numberSchema }), + requests: { + type: "array", + items: obj({ + method: stringSchema, + path: stringSchema, + url: stringSchema, + body: stringSchema, + expectStatus: integerSchema, + expectBody: stringSchema, + }), + }, + path: stringSchema, + url: stringSchema, + script: { + type: "array", + items: obj( + { + action: { type: "string", enum: ["click", "key", "type", "wait"] }, + selector: stringSchema, + key: stringSchema, + text: stringSchema, + ms: numberSchema, + }, + ["action"], + ), + }, + seconds: numberSchema, + probes: { + type: "array", + items: obj({ name: stringSchema, expr: stringSchema }, ["name", "expr"]), + }, + checks: stringArraySchema, + }, + ["kind"], + ), + ], // ── terminal verbs ─────────────────────────────────────────────────────── // The OpenAI adapter overrides these with hand-tuned schemas (see diff --git a/src/prompt/default-tool-descriptors-b.ts b/src/prompt/default-tool-descriptors-b.ts index a6434557..090deb14 100644 --- a/src/prompt/default-tool-descriptors-b.ts +++ b/src/prompt/default-tool-descriptors-b.ts @@ -225,6 +225,31 @@ export const DEFAULT_TOOL_DESCRIPTORS_B: readonly ToolDescriptor[] = [ "{ server: string, name: string, arguments?: Record }", tier: "rare", }, + { + // Frequent tier: the review step of a fan-out reaches for it, and a + // reviewer that has to `tool.view` first reviews less. Read-only, so + // the fusion orchestrator gate lets it through (decision D1). + name: "verify.syntax", + summary: + "Syntax-check files, one checker per file by extension (.js/.json in-process then node --check, .ts via the project's tsc, .py, .sh, .html inline scripts + a warning for content after , .css braces). Read-only. Reports a file with no checker as unchecked — never as passing.", + argsSchema: "{ files: string[] }", + }, + { + // Frequent tier for the same reason, and with an example: the + // runtime failures no syntax check sees (a throw on load, a dead + // button, a probe that never moves) are what this catches, and a + // page run with two probes is not something a model first-shots + // from a one-line manifest. + name: "verify.run", + summary: + "Run what was built against a throwaway copy of the working directory (nothing it writes reaches the workspace; may require approval). kind 'command': a test runner, compiler or script. kind 'service': start a server, wait for a port/url, send requests. kind 'page': a local HTML file or url in a headless browser — scripted input, then `seconds` of runtime with `probes` sampled; collects uncaught errors, console errors and failed getElementById/querySelector lookups. `checks` are assertions on the result. Default `network: false` (a soft proxy block, not a sandbox).", + argsSchema: + "{ kind: 'command' | 'service' | 'page', cmd?: string, args?: string[], start?: { cmd: string, args?: string[] }, ready?: { port?: number, url?: string, timeoutMs?: number }, requests?: [{ method?: string, path?: string, url?: string, body?: string, expectStatus?: number, expectBody?: string }], path?: string, url?: string, script?: [{ action: 'click' | 'key' | 'type' | 'wait', selector?: string, key?: string, text?: string, ms?: number }], seconds?: number /* page runtime, default 5 */, probes?: [{ name: string, expr: string /* JS expression */ }], checks?: string[] /* 'exit 0' | 'exit != 0' | 'stdout contains \"x\"' | 'stderr not contains \"x\"' | 'status 200' | 'no errors' | 'missing selectors 0' | 'probe decreases|increases|equals |reaches |stays ' */, cwd?: string, env?: Record, timeoutMs?: number, network?: boolean }", + examples: [ + '{"kind":"page","path":"index.html","script":[{"action":"click","selector":"#launch"},{"action":"key","key":"ArrowLeft"}],"seconds":6,"probes":[{"name":"lives","expr":"window.game.lives"},{"name":"score","expr":"window.game.score"}],"checks":["no errors","missing selectors 0","probe score increases"]}', + '{"kind":"command","cmd":"npm","args":["test"],"checks":["exit 0","stdout not contains \\"failed\\""]}', + ], + }, { // `frequent` tier and a full example: the whole point of the tool is // that a cloud orchestrator reaches for it instead of doing the bulk diff --git a/src/prompt/fusion-guidance.test.ts b/src/prompt/fusion-guidance.test.ts index dc85dd81..0a7b8377 100644 --- a/src/prompt/fusion-guidance.test.ts +++ b/src/prompt/fusion-guidance.test.ts @@ -208,7 +208,10 @@ describe("the machine facts in the ### fusion block", () => { // the price of sending work that cannot succeed without the // conversation the worker will not have. expect(FUSION_GUIDANCE).toMatch(/prefer sending more/i); - expect(FUSION_GUIDANCE).toContain("independent, self-contained parts"); + expect(FUSION_GUIDANCE).toContain("independent parts"); + // And to check a fan-out before accepting it, with the tools that can. + expect(FUSION_GUIDANCE).toContain("`verify.syntax` on the declared files"); + expect(FUSION_GUIDANCE).toContain("`verify.run` on what the request must do"); expect(FUSION_GUIDANCE).toMatch(/only makes sense with this conversation/i); // And the width is stated as the model's own call. expect(FUSION_GUIDANCE).toMatch(/You choose `maxWorkers`/); diff --git a/src/prompt/fusion-guidance.ts b/src/prompt/fusion-guidance.ts index d3275156..11b1514c 100644 --- a/src/prompt/fusion-guidance.ts +++ b/src/prompt/fusion-guidance.ts @@ -63,13 +63,14 @@ export function isFusionActive( */ export const FUSION_GUIDANCE = [ "You orchestrate the workers: read enough to decide, plan, delegate the doing, review what comes back.", - "Plan in the open, then delegate in the same turn — never stop at the plan: list the independent, self-contained parts, sized so a big one gets its own worker and small ones share.", - "One task per part, in one `fusion.delegate` call. List the paths a task will produce in its `files` — the operator is asked once, about those directories, and that answer is what lets the workers write. Each `instructions` must stand alone: workers see the operator's request, not this chat, and cannot ask you anything.", + "Plan in the open, then delegate in the same turn — never stop at the plan: list the independent parts, sized so a big one gets its own worker and small ones share.", + "One task per part, in one `fusion.delegate` call. List the paths a task will produce in its `files` — the operator is asked once about those directories, and that is what lets the workers write. Each `instructions` must stand alone: workers see the operator's request, not this chat, and cannot ask you.", "You choose `maxWorkers` per call; prefer sending more parts over doing any yourself.", - "Tools that change things are refused for you, always: the workers build, you do not. That is the mode working, not a fault.", + "Tools that change things are refused for you: the workers build, you do not. That is the mode working, not a fault.", "Keep the design and the judgement: read every reply against its brief.", - "Rework goes back out: anything `failed`, `cancelled`, `needs_orchestrator` or just not good enough is another `fusion.delegate` saying what was wrong and what good looks like. Keep going until you would sign off on it.", - "Yours alone: the decision you were asked for, a part that only makes sense with this conversation in front of it, and anything needing operator approval — workers cannot reach the user.", + "Before accepting a fan-out, check it: `verify.syntax` on the declared files and `verify.run` on what the request must do.", + "Rework goes back out: anything `failed`, `cancelled`, `needs_orchestrator` or not good enough is another `fusion.delegate` saying what was wrong and what good looks like.", + "Yours alone: the decision you were asked for, a part that only makes sense with this conversation in front of it, and anything needing operator approval.", "Call `fusion.delegate` on its own, never alongside other tool calls — it runs several turns internally.", ].join("\n"); diff --git a/src/runtime/bootstrap.ts b/src/runtime/bootstrap.ts index a74b577f..e8e34786 100644 --- a/src/runtime/bootstrap.ts +++ b/src/runtime/bootstrap.ts @@ -63,6 +63,7 @@ import { buildBrowserTools } from "../tools/browser/index.js"; import { PlaywrightBackend } from "../tools/browser/playwright-backend.js"; import type { BrowserBackend } from "../tools/browser/browser-backend.js"; import { registerOsTools } from "../tools/os/index.js"; +import { registerVerifyTools, runChecks } from "../tools/verify/index.js"; import { registerGithubTools } from "../tools/github/index.js"; import { resolveGithubToken } from "../github/index.js"; import { registerSkillTools } from "../tools/skill/index.js"; @@ -1445,6 +1446,14 @@ export async function createAgentRuntime( isGitRemoteSyncEnabled: () => getConfig().git.remoteSync, }, }); + // The read-only `verify.*` family: syntax per file, and (below) a + // command / service / page run against a throwaway copy of the + // working directory. Registered next to the OS tools because it is + // the review half of what they build. + registerVerifyTools(toolRegistry, { + ...dangerous, + config: { browser: config.browser }, + }); // Always registered; each call resolves `GITHUB_TOKEN` afresh so a // token saved in the Integrations hub works on the next turn. The // descriptors, by contrast, are gated on the token (see @@ -3066,6 +3075,15 @@ export async function createAgentRuntime( emitEvent: emitAgentLoopEventFor, workingDir, outputCharCap: config.agent.batchToolResultCharCap, + // A contract's declared `checks` run through the verify family, + // each on a throwaway copy of the workspace, so a fan-out is judged + // by what its output does, never by what a worker's reply says. + runChecks: (specs, ctx) => + runChecks(specs, { + workingDir: ctx.workingDir, + signal: ctx.signal, + config, + }), logger, }), ); diff --git a/src/tools/fusion/worker-read-scope.ts b/src/tools/fusion/worker-read-scope.ts index 18546e27..f5e25ecb 100644 --- a/src/tools/fusion/worker-read-scope.ts +++ b/src/tools/fusion/worker-read-scope.ts @@ -88,6 +88,14 @@ export const WORKER_READ_TOOL_TARGETS: ReadonlyMap = ...(Array.isArray(args.paths) ? args.paths.map(nonEmpty) : []), ].filter((path): path is string => path !== undefined), ], + [ + // A syntax check reads every file it is handed. + "verify.syntax", + (args) => + (Array.isArray(args.files) ? args.files.map(nonEmpty) : []).filter( + (path): path is string => path !== undefined, + ), + ], ]); /** `https://…`, `data:…` — not a filesystem path, not this module's business. */ diff --git a/src/tools/tool-roles.test.ts b/src/tools/tool-roles.test.ts index 2d0dc5c6..611f43ac 100644 --- a/src/tools/tool-roles.test.ts +++ b/src/tools/tool-roles.test.ts @@ -102,6 +102,10 @@ describe("tool roles", () => { for (const d of DEFAULT_TOOL_DESCRIPTORS) { if (!roleAdmits("orchestrator", d.name)) continue; if (d.name === "fusion.delegate") continue; + // `verify.run` executes a command, so it is approval-gated below + // level 4 — but it runs against a throwaway copy of the workspace, + // which is why the fusion gate admits it as a read (`readonly`). + if (d.name === "verify.run") continue; expect(resourceClassFor(d.name), d.name).toMatch(/^(pure_read|terminal)$/); } }); diff --git a/src/tools/verify/check-css-syntax.test.ts b/src/tools/verify/check-css-syntax.test.ts new file mode 100644 index 00000000..6ae7651f --- /dev/null +++ b/src/tools/verify/check-css-syntax.test.ts @@ -0,0 +1,39 @@ +import { describe, expect, it } from "vitest"; + +import { checkCssSource, CSS_CHECKER } from "./check-css-syntax.js"; + +describe("checkCssSource", () => { + it("passes balanced rules, nested blocks, strings and comments", () => { + const css = ` +/* a comment with a stray } inside */ +.a { color: red; } +@media (max-width: 600px) { .b { content: "{not a brace"; } } +.c::after { content: '}'; } +`; + expect(checkCssSource("x.css", css)).toEqual({ + file: "x.css", + ok: true, + checker: CSS_CHECKER, + }); + }); + + it("names the line of a closing brace with no open block", () => { + const css = ".a { color: red; }\n}\n.b { }\n"; + const out = checkCssSource("x.css", css); + expect(out.ok).toBe(false); + expect(out.error).toBe("unexpected `}` at line 2 (no open block)"); + }); + + it("counts the blocks left open at the end of the file", () => { + const out = checkCssSource("x.css", ".a { .b { color: red; }\n.c {"); + expect(out.ok).toBe(false); + expect(out.error).toBe("2 unclosed `{` at end of file"); + }); + + it("does not let an unterminated string swallow the rest of the file", () => { + // The string ends at the newline (invalid CSS, but the brace count + // must still see the `}` on the next line). + const out = checkCssSource("x.css", ".a { content: \"oops;\n}\n"); + expect(out.ok).toBe(true); + }); +}); diff --git a/src/tools/verify/check-css-syntax.ts b/src/tools/verify/check-css-syntax.ts new file mode 100644 index 00000000..955294f1 --- /dev/null +++ b/src/tools/verify/check-css-syntax.ts @@ -0,0 +1,84 @@ +/** + * CSS: brace balance, outside strings and comments. + * + * Not a CSS parser — a stray `}` or an unclosed rule is what a model + * leaves behind when an edit cuts a block in half, and that is what this + * catches. Property typos and unknown selectors are the browser's to + * ignore, not ours to flag. + */ +import type { SyntaxFileResult } from "./syntax-check-types.js"; + +export const CSS_CHECKER = "css-braces"; + +export function checkCssSource( + file: string, + content: string, +): SyntaxFileResult { + let depth = 0; + let line = 1; + let i = 0; + const n = content.length; + while (i < n) { + const ch = content[i]; + if (ch === "\n") { + line += 1; + i += 1; + } else if (ch === "/" && content[i + 1] === "*") { + const end = content.indexOf("*/", i + 2); + const stop = end === -1 ? n : end + 2; + line += countNewlines(content, i, stop); + i = stop; + } else if (ch === '"' || ch === "'") { + const stop = skipString(content, i, ch); + line += countNewlines(content, i, stop); + i = stop; + } else if (ch === "{") { + depth += 1; + i += 1; + } else if (ch === "}") { + depth -= 1; + if (depth < 0) { + return { + file, + ok: false, + checker: CSS_CHECKER, + error: `unexpected \`}\` at line ${line} (no open block)`, + }; + } + i += 1; + } else { + i += 1; + } + } + if (depth > 0) { + return { + file, + ok: false, + checker: CSS_CHECKER, + error: `${depth} unclosed \`{\` at end of file`, + }; + } + return { file, ok: true, checker: CSS_CHECKER }; +} + +function skipString(text: string, start: number, quote: string): number { + let i = start + 1; + while (i < text.length) { + const ch = text[i]; + if (ch === "\\") { + i += 2; + continue; + } + // An unescaped newline ends a CSS string (it is invalid, but the + // brace count must not run away because of it). + if (ch === quote || ch === "\n") return i + 1; + i += 1; + } + return text.length; +} + +function countNewlines(text: string, from: number, to: number): number { + let count = 0; + for (let i = from; i < to; i += 1) if (text[i] === "\n") count += 1; + return count; +} diff --git a/src/tools/verify/check-html-syntax.test.ts b/src/tools/verify/check-html-syntax.test.ts new file mode 100644 index 00000000..ca97d960 --- /dev/null +++ b/src/tools/verify/check-html-syntax.test.ts @@ -0,0 +1,100 @@ +import { describe, expect, it } from "vitest"; + +import { + checkHtmlSource, + extractInlineScripts, + HTML_CHECKER, + trailingContentWarning, +} from "./check-html-syntax.js"; + +describe("extractInlineScripts", () => { + it("keeps inline JS blocks, skips external and non-JS ones, and records the line", () => { + const html = [ + "", + '', + '', + "", + '', + "", + ].join("\n"); + const { scripts, external } = extractInlineScripts(html); + expect(external).toBe(1); + expect(scripts.map((s) => [s.index, s.line, s.module])).toEqual([ + [3, 4, false], + [4, 7, true], + ]); + expect(scripts[0]?.code.trim()).toBe("var a = 1;"); + }); +}); + +describe("trailingContentWarning", () => { + it("is silent for whitespace after and for fragments without it", () => { + expect(trailingContentWarning("\n\n")).toBeNull(); + expect(trailingContentWarning("
fragment
")).toBeNull(); + }); + + it("names content after the closing tag", () => { + const warning = trailingContentWarning( + "\n```\nleftover text\n```", + ); + expect(warning).toContain("content after "); + expect(warning).toContain("leftover text"); + }); +}); + +describe("checkHtmlSource", () => { + it("passes a page whose inline scripts parse", async () => { + const out = await checkHtmlSource( + "index.html", + "", + ); + expect(out).toEqual({ file: "index.html", ok: true, checker: HTML_CHECKER }); + }); + + it("fails on the block with the error, with the line rebased onto the document", async () => { + const html = [ + "", + "", + "", + "", + ].join("\n"); + const out = await checkHtmlSource("index.html", html); + expect(out.ok).toBe(false); + // The label carries the ``). + expect(out.error).toContain("inline script #2 (line 5)"); + expect(out.error).toMatch(/SyntaxError: .* \(line 7\)/); + }); + + it("checks an inline ES module through node --check rather than skipping it", async () => { + const html = + ''; + const out = await checkHtmlSource("index.html", html); + expect(out.ok).toBe(false); + expect(out.error).toMatch(/SyntaxError/); + }); + + it("gives no verdict for a page with no inline script, and never a pass", async () => { + const out = await checkHtmlSource( + "index.html", + '', + ); + expect(out.ok).toBeNull(); + expect(out.error).toBe("no inline scripts to check (1 external)"); + }); + + it("attaches the trailing-content warning without changing the verdict", async () => { + const out = await checkHtmlSource( + "index.html", + "\nstray", + ); + expect(out.ok).toBe(true); + expect(out.warning).toContain("content after "); + }); +}); diff --git a/src/tools/verify/check-html-syntax.ts b/src/tools/verify/check-html-syntax.ts new file mode 100644 index 00000000..f1f4b264 --- /dev/null +++ b/src/tools/verify/check-html-syntax.ts @@ -0,0 +1,145 @@ +/** + * HTML: every inline ` + +`, + ); + const out = await runVerify( + { + kind: "page", + path: "index.html", + script: [{ action: "click", selector: "#launch" }, { action: "wait", ms: 100 }], + seconds: 1, + probes: [{ name: "score", expr: "window.game.score" }, { name: "launched", expr: "window.game.launched" }], + checks: ["no errors", "missing selectors 2", "probe score increases", "probe launched stays true"], + }, + ctx(), + ); + expect(out.errors).toEqual(["Error: boom on load"]); + expect(out.consoleWarnings).toEqual(["just a warning"]); + expect(out.missingSelectors).toEqual(["# does-not-exist", "querySelector .nope"]); + expect(out.ok).toBe(false); + expect(out.checks?.map((c) => c.ok)).toEqual([false, true, true, true]); + const score = out.probes?.score ?? []; + expect(score.length).toBeGreaterThan(1); + expect(score.length).toBeLessThanOrEqual(40); + expect(out.summary).toContain("uncaught errors (1)"); + }, 60_000); +}); + +describe("runChecks", () => { + it("runs every spec in order and is ok only when all are", async () => { + const out = await runChecks( + [ + { kind: "command", cmd: NODE, args: ["-e", "process.exit(0)"], checks: ["exit 0"] }, + { kind: "command", cmd: NODE, args: ["-e", "process.exit(1)"], checks: ["exit 0"] }, + { kind: "nonsense" }, + ], + ctx(), + ); + expect(out.ok).toBe(false); + expect(out.results.map((r) => r.ok)).toEqual([true, false, false]); + expect(out.results[2]?.error).toContain("`kind`"); + const good = await runChecks([{ kind: "command", cmd: NODE, args: ["-v"], checks: ["exit 0"] }], ctx()); + expect(good.ok).toBe(true); + }); +}); diff --git a/src/tools/verify/run-verify.ts b/src/tools/verify/run-verify.ts new file mode 100644 index 00000000..2ebdc4c9 --- /dev/null +++ b/src/tools/verify/run-verify.ts @@ -0,0 +1,160 @@ +/** + * `runVerify` — the programmatic face of `verify.run`, shared by the + * tool and by the fan-out contract (`runChecks`). + * + * One result shape for the three kinds: `ok`, `kind`, `isolated`, + * `durationMs`, the kind's own fields, the `checks` outcomes and a + * `summary` with failures first. `ok` means the run itself went well + * AND every check passed; a check that fails to parse fails the run. + */ +import { resolve, relative, isAbsolute } from "node:path"; + +import { getConfig, type AtomicAgentConfig } from "../../config/index.js"; +import type { ProbeSample } from "./page-probe-script.js"; +import { runCommandKind } from "./run-command-kind.js"; +import { type BrowserLauncher, defaultBrowserLauncher, runPageKind } from "./run-page-kind.js"; +import { type FetchLike, type RequestOutcome, runServiceKind } from "./run-service-kind.js"; +import { type CheckOutcome, type CheckSubject, evaluateChecks } from "./verify-checks.js"; +import { parseVerifyRunArgs, type VerifyRunArgs, type VerifyRunKind } from "./verify-run-args.js"; +import { renderVerifyRunSummary } from "./verify-run-summary.js"; +import { createVerifyWorkspace, type VerifyWorkspace } from "./verify-workspace-copy.js"; + +export interface VerifyRunResult { + readonly ok: boolean; + readonly kind: VerifyRunKind; + readonly isolated: boolean; + readonly durationMs: number; + /** A run that could not happen (no browser, bad cwd, spawn failure). */ + readonly error?: string; + readonly commandLine?: string; + readonly exitCode?: number | null; + readonly timedOut?: boolean; + readonly stdoutTail?: string; + readonly stderrTail?: string; + readonly ready?: boolean; + readonly readyMs?: number | null; + readonly requests?: readonly RequestOutcome[]; + readonly errors?: readonly string[]; + readonly consoleErrors?: readonly string[]; + readonly consoleWarnings?: readonly string[]; + readonly requestFailures?: readonly string[]; + readonly missingSelectors?: readonly string[]; + readonly probes?: Readonly>; + readonly checks?: readonly CheckOutcome[]; + readonly summary: string; +} + +export interface VerifyRunContext { + readonly workingDir: string; + /** Browser settings for page runs; `getConfig().browser` when omitted. */ + readonly config?: Pick; + readonly signal?: AbortSignal; + /** Test seams. */ + readonly launchBrowser?: BrowserLauncher; + readonly fetchImpl?: FetchLike; + readonly workspace?: (workingDir: string) => Promise; +} + +/** `cwd` relative to the copy, refused when it escapes it. */ +function resolveRunCwd(root: string, cwd: string | undefined): string { + if (cwd === undefined) return root; + const target = resolve(root, cwd); + const rel = relative(root, target); + if (rel.startsWith("..") || isAbsolute(rel)) { + throw new Error(`verify.run: \`cwd\` must stay inside the working directory (got ${cwd})`); + } + return target; +} + +function finish( + base: Omit, + subject: CheckSubject, + specs: readonly string[] | undefined, + runOk: boolean, + started: number, +): VerifyRunResult { + // A run that did not happen (no browser, a server that never came up, + // a command that could not start) evaluates nothing: `no errors` over + // an empty error list would be the fake pass this tool exists to + // refuse. + const checks = + specs === undefined || specs.length === 0 + ? undefined + : base.error !== undefined + ? specs.map((check) => ({ check, ok: false, detail: `not evaluated: ${base.error}` })) + : evaluateChecks(specs, subject); + const ok = runOk && (checks?.every((c) => c.ok) ?? true); + const withoutSummary = { ...base, ok, durationMs: Date.now() - started, ...(checks === undefined ? {} : { checks }) }; + return { ...withoutSummary, summary: renderVerifyRunSummary({ ...withoutSummary, summary: "" }) }; +} + +async function runInWorkspace(args: VerifyRunArgs, dir: string, isolated: boolean, ctx: VerifyRunContext, started: number): Promise { + const cwd = resolveRunCwd(dir, args.cwd); + const common = { kind: args.kind, isolated }; + if (args.kind === "command") { + const out = await runCommandKind(args, { cwd, signal: ctx.signal }); + const runOk = out.spawnError === undefined && !out.timedOut && out.exitCode === 0; + return finish( + { ...common, commandLine: out.commandLine, exitCode: out.exitCode, timedOut: out.timedOut, stdoutTail: out.stdoutTail, stderrTail: out.stderrTail, ...(out.spawnError === undefined ? {} : { error: `could not start \`${out.commandLine}\`: ${out.spawnError}` }) }, + { kind: "command", exitCode: out.exitCode, timedOut: out.timedOut, stdout: out.stdout, stderr: out.stderr }, + args.checks, runOk, started, + ); + } + if (args.kind === "service") { + const out = await runServiceKind(args, { cwd, signal: ctx.signal, fetchImpl: ctx.fetchImpl }); + const runOk = out.ready && out.requests.every((q) => q.ok); + return finish( + { ...common, commandLine: out.commandLine, exitCode: out.exitCode, timedOut: out.timedOut, ready: out.ready, readyMs: out.readyMs, requests: out.requests, stdoutTail: out.stdoutTail, stderrTail: out.stderrTail, ...(out.readyError === undefined ? {} : { error: out.readyError }) }, + { kind: "service", exitCode: out.exitCode, timedOut: out.timedOut, stdout: out.stdout, stderr: out.stderr, requests: out.requests }, + args.checks, runOk, started, + ); + } + const launch = + ctx.launchBrowser ?? + defaultBrowserLauncher(ctx.config ?? { browser: getConfig().browser }); + const out = await runPageKind(args, { cwd, signal: ctx.signal, launch }); + const runOk = out.launched && out.loaded && out.errors.length === 0 && out.consoleErrors.length === 0; + return finish( + { ...common, errors: out.errors, consoleErrors: out.consoleErrors, consoleWarnings: out.consoleWarnings, requestFailures: out.requestFailures, missingSelectors: out.missingSelectors, probes: out.probes, ...(out.error === undefined ? {} : { error: out.error }) }, + { kind: "page", errors: out.errors, consoleErrors: out.consoleErrors, missingSelectors: out.missingSelectors, probes: out.probes }, + args.checks, runOk, started, + ); +} + +/** Run one spec against a throwaway copy of `ctx.workingDir`. */ +export async function runVerify( + rawArgs: VerifyRunArgs | Record, + ctx: VerifyRunContext, +): Promise { + const started = Date.now(); + const args = parseVerifyRunArgs(rawArgs as Record); + const workspace = await (ctx.workspace ?? createVerifyWorkspace)(ctx.workingDir); + try { + return await runInWorkspace(args, workspace.dir, workspace.isolated, ctx, started); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + const base = { kind: args.kind, isolated: workspace.isolated, error }; + return finish(base, { kind: args.kind }, args.checks, false, started); + } finally { + await workspace.cleanup(); + } +} + +/** The fan-out contract's checks: every spec in order, `ok` when all are. */ +export async function runChecks( + specs: readonly (VerifyRunArgs | Record)[], + ctx: VerifyRunContext, +): Promise<{ ok: boolean; results: VerifyRunResult[] }> { + const results: VerifyRunResult[] = []; + for (const spec of specs) { + if (ctx.signal?.aborted) break; + try { + results.push(await runVerify(spec, ctx)); + } catch (err) { + const error = err instanceof Error ? err.message : String(err); + const kind = (typeof spec.kind === "string" ? spec.kind : "command") as VerifyRunKind; + results.push({ ok: false, kind, isolated: true, durationMs: 0, error, summary: `verify.run ${kind}: FAILED\n${error}` }); + } + } + return { ok: results.length === specs.length && results.every((r) => r.ok), results }; +} diff --git a/src/tools/verify/spawn-verify-process.ts b/src/tools/verify/spawn-verify-process.ts new file mode 100644 index 00000000..1edc8c4d --- /dev/null +++ b/src/tools/verify/spawn-verify-process.ts @@ -0,0 +1,174 @@ +/** + * The child process a `command` or `service` run owns. + * + * Spawned `detached` so it leads its own process group, and the whole + * group is killed with SIGKILL when the run ends — on timeout, on abort, + * and when the command itself finishes, because a test script that + * leaves a server behind must not leave it behind in our process tree. + * + * `network: false` is a soft block: `HTTP_PROXY` / `HTTPS_PROXY` / + * `ALL_PROXY` point at an unreachable local port and `NO_PROXY` is + * emptied, which stops every proxy-honouring client (curl, npm, pip, + * fetch under undici's env proxy, most SDKs). It is not a sandbox: a + * raw socket ignores it. + */ +import { spawn, type ChildProcess } from "node:child_process"; + +import { killProcessTree } from "../../sandbox/kill-process-tree.js"; +import { buildSubshellInvocation } from "../../sandbox/shell-invocation.js"; +import { needsShellInterpretation } from "../os/shell.js"; + +export const OUTPUT_TAIL_CHARS = 8_000; +/** Kept for `stdout contains` checks; past this the head is dropped. */ +const OUTPUT_KEEP_CHARS = 1024 * 1024; + +export class TailBuffer { + private text = ""; + private dropped = false; + + append(chunk: Buffer | string): void { + this.text += typeof chunk === "string" ? chunk : chunk.toString("utf8"); + if (this.text.length > OUTPUT_KEEP_CHARS) { + this.text = this.text.slice(-OUTPUT_KEEP_CHARS); + this.dropped = true; + } + } + + /** Everything kept (up to 1 MB, head dropped first). */ + all(): string { + return this.text; + } + + /** The last 8,000 chars — tail first, that is where the failure is. */ + tail(): string { + const clipped = this.text.length > OUTPUT_TAIL_CHARS || this.dropped; + const tail = this.text.slice(-OUTPUT_TAIL_CHARS); + return clipped ? `…${tail}` : tail; + } +} + +export const NETWORK_BLOCK_PROXY = "http://127.0.0.1:9"; + +export function networkBlockEnv(): Record { + return { + HTTP_PROXY: NETWORK_BLOCK_PROXY, + HTTPS_PROXY: NETWORK_BLOCK_PROXY, + ALL_PROXY: NETWORK_BLOCK_PROXY, + http_proxy: NETWORK_BLOCK_PROXY, + https_proxy: NETWORK_BLOCK_PROXY, + all_proxy: NETWORK_BLOCK_PROXY, + NO_PROXY: "", + no_proxy: "", + }; +} + +export interface SpawnVerifyOptions { + cwd: string; + env?: Readonly>; + network: boolean; + timeoutMs: number; + signal?: AbortSignal; +} + +export interface VerifyProcessExit { + readonly exitCode: number | null; + readonly signal: NodeJS.Signals | null; + readonly timedOut: boolean; + readonly aborted: boolean; + /** `spawn` itself failed (ENOENT and friends). */ + readonly spawnError?: string; +} + +export interface VerifyProcess { + readonly child: ChildProcess; + readonly commandLine: string; + readonly stdout: TailBuffer; + readonly stderr: TailBuffer; + readonly exited: Promise; + /** SIGKILL the whole process group; idempotent. */ + killGroup(): void; +} + +/** The argv actually spawned: direct, or through the OS subshell for a command line. */ +export function resolveInvocation( + cmd: string, + args: readonly string[], +): { command: string; args: string[]; commandLine: string } { + const commandLine = [cmd, ...args].join(" "); + if (needsShellInterpretation(cmd, args)) { + const shell = buildSubshellInvocation(commandLine); + return { command: shell.command, args: shell.args, commandLine }; + } + return { command: cmd, args: [...args], commandLine }; +} + +export function spawnVerifyProcess( + cmd: string, + args: readonly string[], + options: SpawnVerifyOptions, +): VerifyProcess { + const invocation = resolveInvocation(cmd, args); + const detached = process.platform !== "win32"; + const child = spawn(invocation.command, invocation.args, { + cwd: options.cwd, + env: { + ...process.env, + ...(options.network ? {} : networkBlockEnv()), + ...(options.env ?? {}), + }, + detached, + stdio: ["ignore", "pipe", "pipe"], + windowsHide: true, + }); + const stdout = new TailBuffer(); + const stderr = new TailBuffer(); + child.stdout?.on("data", (chunk: Buffer) => stdout.append(chunk)); + child.stderr?.on("data", (chunk: Buffer) => stderr.append(chunk)); + + let killed = false; + let timedOut = false; + let aborted = false; + const killGroup = (): void => { + if (killed) return; + killed = true; + if (detached && typeof child.pid === "number") { + try { + process.kill(-child.pid, "SIGKILL"); + return; + } catch { + // Group already gone, or not a leader after all — fall through. + } + } + killProcessTree(child, { force: true }); + }; + const timer = setTimeout(() => { + timedOut = true; + killGroup(); + }, options.timeoutMs); + const onAbort = (): void => { + aborted = true; + killGroup(); + }; + options.signal?.addEventListener("abort", onAbort, { once: true }); + + const exited = new Promise((resolve) => { + let settled = false; + const finish = (exit: VerifyProcessExit): void => { + if (settled) return; + settled = true; + clearTimeout(timer); + options.signal?.removeEventListener("abort", onAbort); + // Whatever the command left running in its group dies with it. + killGroup(); + resolve(exit); + }; + child.on("error", (err) => { + finish({ exitCode: null, signal: null, timedOut, aborted, spawnError: err.message }); + }); + child.on("close", (code, signal) => { + finish({ exitCode: code, signal, timedOut, aborted }); + }); + }); + + return { child, commandLine: invocation.commandLine, stdout, stderr, exited, killGroup }; +} diff --git a/src/tools/verify/syntax-check-types.ts b/src/tools/verify/syntax-check-types.ts new file mode 100644 index 00000000..50f0d2de --- /dev/null +++ b/src/tools/verify/syntax-check-types.ts @@ -0,0 +1,38 @@ +/** + * One file's verdict from `verify.syntax`. + * + * `ok` has three values on purpose. `true` and `false` are verdicts; + * `null` is the absence of one — no checker for the extension, the + * checker's binary is missing, or the checker could not judge the file + * (an ES module under a plain-script parser). A file with no verdict is + * reported as unchecked and never counts as passing: a green result over + * a file nobody looked at is worse than no check at all. + */ +export interface SyntaxFileResult { + /** The path as the caller gave it. */ + readonly file: string; + readonly ok: boolean | null; + /** The checker that produced the verdict, or `"none"`. */ + readonly checker: string; + /** The failure, or why there is no verdict. */ + readonly error?: string; + /** Advisory — never changes `ok` (content after ``, say). */ + readonly warning?: string; +} + +/** Longest error text kept per file; checkers are chatty, prompts are not. */ +export const SYNTAX_ERROR_MAX_CHARS = 600; + +/** The last non-blank lines of a checker's stderr, capped. */ +export function tailOfOutput(text: string, lines = 5): string { + const kept = text + .replace(/\r\n/g, "\n") + .split("\n") + .map((line) => line.trimEnd()) + .filter((line) => line.trim().length > 0) + .slice(-lines) + .join("\n"); + return kept.length > SYNTAX_ERROR_MAX_CHARS + ? `…${kept.slice(-SYNTAX_ERROR_MAX_CHARS)}` + : kept; +} diff --git a/src/tools/verify/verify-checks.test.ts b/src/tools/verify/verify-checks.test.ts new file mode 100644 index 00000000..ef7d5dc9 --- /dev/null +++ b/src/tools/verify/verify-checks.test.ts @@ -0,0 +1,114 @@ +import { describe, expect, it } from "vitest"; + +import { + type CheckSubject, + evaluateCheck, + evaluateChecks, + tokenizeCheck, +} from "./verify-checks.js"; + +const COMMAND: CheckSubject = { + kind: "command", + exitCode: 1, + stdout: "12 passed\n2 failed\n", + stderr: "", +}; + +const PAGE: CheckSubject = { + kind: "page", + errors: [], + consoleErrors: ["TypeError: x is null"], + missingSelectors: ["# launch-btn"], + probes: { + lives: [[0, 3], [250, 3], [500, 2], [750, 1]], + score: [[0, 0], [250, 10], [500, 20]], + state: [[0, "menu"], [250, "menu"], [500, "playing"]], + }, +}; + +describe("tokenizeCheck", () => { + it("keeps quoted strings whole and unescapes them", () => { + expect(tokenizeCheck('stdout contains "all tests passed"')).toEqual([ + "stdout", + "contains", + "all tests passed", + ]); + expect(tokenizeCheck("stderr not contains 'a \\'b\\' c'")).toEqual([ + "stderr", + "not", + "contains", + "a 'b' c", + ]); + }); +}); + +describe("evaluateCheck", () => { + it("exit: equality and inequality, with the actual code in the detail", () => { + expect(evaluateCheck("exit 0", COMMAND)).toMatchObject({ ok: false, detail: "exit 1" }); + expect(evaluateCheck("exit != 0", COMMAND)).toMatchObject({ ok: true }); + expect(evaluateCheck("exit == 1", COMMAND)).toMatchObject({ ok: true }); + expect(evaluateCheck("exit 0", { kind: "command", exitCode: null, timedOut: true })).toMatchObject({ + ok: false, + detail: "killed (timed out)", + }); + expect(evaluateCheck("exit 0", PAGE).ok).toBe(false); + }); + + it("stdout / stderr contains, negated or not, over the full output", () => { + expect(evaluateCheck('stdout contains "12 passed"', COMMAND).ok).toBe(true); + expect(evaluateCheck('stdout not contains "failed"', COMMAND).ok).toBe(false); + expect(evaluateCheck('stderr not contains "Error"', COMMAND).ok).toBe(true); + expect(evaluateCheck("stdout contains", COMMAND).detail).toContain("unknown check"); + }); + + it("status: every request must answer with the code", () => { + const service: CheckSubject = { kind: "service", requests: [{ status: 200 }, { status: 200 }] }; + expect(evaluateCheck("status 200", service).ok).toBe(true); + expect(evaluateCheck("status 200", { kind: "service", requests: [{ status: 200 }, { status: 500 }] })).toMatchObject({ + ok: false, + detail: "statuses: 200, 500", + }); + expect(evaluateCheck("status 200", { kind: "service", requests: [] }).detail).toBe("no requests were made"); + }); + + it("no errors and missing selectors are page checks", () => { + expect(evaluateCheck("no errors", PAGE)).toMatchObject({ + ok: false, + detail: "0 uncaught error(s), 1 console error(s)", + }); + expect(evaluateCheck("no errors", { ...PAGE, consoleErrors: [] }).ok).toBe(true); + expect(evaluateCheck("no errors", COMMAND).ok).toBe(false); + expect(evaluateCheck("missing selectors 0", PAGE)).toMatchObject({ ok: false }); + expect(evaluateCheck("missing selectors 1", PAGE).ok).toBe(true); + expect(evaluateCheck("missing selectors 0", COMMAND).ok).toBe(false); + }); + + it("probe trends and values", () => { + expect(evaluateCheck("probe lives decreases", PAGE)).toMatchObject({ ok: true, detail: "lives: 3 → 1 over 4 samples" }); + expect(evaluateCheck("probe lives increases", PAGE).ok).toBe(false); + expect(evaluateCheck("probe score increases", PAGE).ok).toBe(true); + expect(evaluateCheck("probe score equals 20", PAGE).ok).toBe(true); + expect(evaluateCheck("probe score equals 10", PAGE).ok).toBe(false); + expect(evaluateCheck("probe state reaches playing", PAGE).ok).toBe(true); + expect(evaluateCheck('probe state reaches "over"', PAGE)).toMatchObject({ ok: false }); + expect(evaluateCheck("probe state stays menu", PAGE)).toMatchObject({ ok: false, detail: "state was playing at sample 2" }); + expect(evaluateCheck("probe lives stays 3", { ...PAGE, probes: { lives: [[0, 3], [250, 3]] } }).ok).toBe(true); + expect(evaluateCheck("probe state decreases", PAGE)).toMatchObject({ ok: false }); + expect(evaluateCheck("probe nothing decreases", PAGE).detail).toBe('no probe named "nothing"'); + }); + + it("fails unknown syntax rather than passing it", () => { + for (const spec of ["passes", "exit", "probe", "no", "no errors please", "probe x between 1 2"]) { + const out = evaluateCheck(spec, PAGE); + expect(out.ok, spec).toBe(false); + expect(out.detail, spec).toContain("unknown check"); + } + }); + + it("evaluateChecks keeps the order of the specs", () => { + expect(evaluateChecks(["exit != 0", "exit 0"], COMMAND).map((c) => [c.check, c.ok])).toEqual([ + ["exit != 0", true], + ["exit 0", false], + ]); + }); +}); diff --git a/src/tools/verify/verify-checks.ts b/src/tools/verify/verify-checks.ts new file mode 100644 index 00000000..ac3970f6 --- /dev/null +++ b/src/tools/verify/verify-checks.ts @@ -0,0 +1,183 @@ +/** + * The `checks` assertion language of `verify.run`. + * + * exit 0 | exit != 0 | exit == 0 + * stdout contains "x" | stdout not contains "x" (stderr likewise) + * status 200 every request answered with that status + * no errors page: no uncaught errors, no console errors + * missing selectors 0 page: that many failed lookups + * probe decreases | increases + * probe equals | reaches | stays + * + * Small on purpose: a check is a sentence the brief's author can write + * and the model can read back. Anything else fails as `unknown check` + * rather than passing by accident. + */ + +export interface CheckOutcome { + readonly check: string; + readonly ok: boolean; + readonly detail: string; +} + +/** What a check may look at — the union of every runner's outcome. */ +export interface CheckSubject { + readonly kind: "command" | "service" | "page"; + readonly exitCode?: number | null; + readonly timedOut?: boolean; + /** Full captured output (capped), not just the tail in the result. */ + readonly stdout?: string; + readonly stderr?: string; + readonly requests?: readonly { readonly status: number | null }[]; + readonly errors?: readonly string[]; + readonly consoleErrors?: readonly string[]; + readonly missingSelectors?: readonly string[]; + readonly probes?: Readonly>; +} + +const PROBE_OPS: ReadonlySet = new Set(["decreases", "increases", "equals", "reaches", "stays"]); + +/** Split on whitespace, keeping double- or single-quoted runs whole. */ +export function tokenizeCheck(spec: string): string[] { + const tokens: string[] = []; + const re = /"((?:[^"\\]|\\.)*)"|'((?:[^'\\]|\\.)*)'|(\S+)/g; + for (const m of spec.matchAll(re)) { + const quoted = m[1] ?? m[2]; + tokens.push(quoted === undefined ? m[3]! : quoted.replace(/\\(.)/g, "$1")); + } + return tokens; +} + +function parseValue(raw: string): unknown { + try { + return JSON.parse(raw); + } catch { + return raw; + } +} + +function same(a: unknown, b: unknown): boolean { + if (a === b) return true; + if (typeof a === "number" && typeof b === "string") return String(a) === b; + if (typeof a === "string" && typeof b === "number") return a === String(b); + return JSON.stringify(a) === JSON.stringify(b); +} + +function show(value: unknown): string { + const text = typeof value === "string" ? value : JSON.stringify(value); + return text === undefined ? "undefined" : text.length > 60 ? `${text.slice(0, 57)}…` : text; +} + +function outcome(check: string, ok: boolean, detail: string): CheckOutcome { + return { check, ok, detail }; +} + +function checkExit(check: string, tokens: string[], s: CheckSubject): CheckOutcome { + const [, a, b] = tokens; + const negated = a === "!="; + const wanted = Number.parseInt((negated || a === "==" ? b : a) ?? "", 10); + if (!Number.isInteger(wanted)) return outcome(check, false, "unknown check: expected `exit `"); + if (s.exitCode === undefined) return outcome(check, false, `no exit code for a ${s.kind} run`); + const actual = s.exitCode === null ? (s.timedOut ? "killed (timed out)" : "killed by signal") : `exit ${s.exitCode}`; + const ok = negated ? s.exitCode !== wanted : s.exitCode === wanted; + return outcome(check, ok, actual); +} + +function checkStream(check: string, tokens: string[], s: CheckSubject): CheckOutcome { + const stream = tokens[0] as "stdout" | "stderr"; + const negated = tokens[1] === "not"; + const verb = negated ? tokens[2] : tokens[1]; + const needle = negated ? tokens[3] : tokens[2]; + if (verb !== "contains" || needle === undefined) { + return outcome(check, false, `unknown check: expected \`${stream} [not] contains "x"\``); + } + const text = s[stream]; + if (text === undefined) return outcome(check, false, `no ${stream} for a ${s.kind} run`); + const found = text.includes(needle); + return outcome(check, negated ? !found : found, found ? `${stream} contains ${show(needle)}` : `${stream} does not contain ${show(needle)}`); +} + +function checkStatus(check: string, tokens: string[], s: CheckSubject): CheckOutcome { + const wanted = Number.parseInt(tokens[1] ?? "", 10); + if (!Number.isInteger(wanted)) return outcome(check, false, "unknown check: expected `status `"); + if (s.requests === undefined || s.requests.length === 0) return outcome(check, false, "no requests were made"); + const statuses = s.requests.map((r) => r.status ?? "no response"); + const ok = statuses.every((st) => st === wanted); + return outcome(check, ok, `statuses: ${statuses.join(", ")}`); +} + +function checkNoErrors(check: string, s: CheckSubject): CheckOutcome { + if (s.kind !== "page") return outcome(check, false, "`no errors` applies to page runs"); + const errors = s.errors?.length ?? 0; + const consoleErrors = s.consoleErrors?.length ?? 0; + return outcome(check, errors === 0 && consoleErrors === 0, `${errors} uncaught error(s), ${consoleErrors} console error(s)`); +} + +function checkMissing(check: string, tokens: string[], s: CheckSubject): CheckOutcome { + const wanted = Number.parseInt(tokens[2] ?? "", 10); + if (tokens[1] !== "selectors" || !Number.isInteger(wanted)) { + return outcome(check, false, "unknown check: expected `missing selectors `"); + } + if (s.missingSelectors === undefined) return outcome(check, false, "selector lookups are recorded for page runs only"); + const list = s.missingSelectors; + return outcome(check, list.length === wanted, list.length === 0 ? "no failed lookups" : `${list.length} failed lookup(s): ${list.slice(0, 5).join(", ")}`); +} + +function checkProbe(check: string, tokens: string[], s: CheckSubject): CheckOutcome { + const [, name, op, rawValue] = tokens; + if (name === undefined || op === undefined || !PROBE_OPS.has(op)) { + return outcome(check, false, "unknown check: expected `probe decreases|increases|equals |reaches |stays `"); + } + const samples = s.probes?.[name]; + if (samples === undefined) return outcome(check, false, `no probe named ${JSON.stringify(name)}`); + const values = samples.map(([, v]) => v); + if (values.length === 0) return outcome(check, false, `probe ${name} has no samples`); + const first = values[0]; + const last = values[values.length - 1]; + const trend = `${name}: ${show(first)} → ${show(last)} over ${values.length} samples`; + if (op === "decreases" || op === "increases") { + if (typeof first !== "number" || typeof last !== "number") return outcome(check, false, `${trend} (not numeric)`); + return outcome(check, op === "decreases" ? last < first : last > first, trend); + } + if (rawValue === undefined) return outcome(check, false, `unknown check: \`probe ${name} ${op}\` needs a value`); + const wanted = parseValue(rawValue); + if (op === "equals") return outcome(check, same(last, wanted), `${name} ended at ${show(last)}`); + if (op === "reaches") { + const hit = values.findIndex((v) => same(v, wanted)); + return outcome(check, hit !== -1, hit === -1 ? `${name} never reached ${show(wanted)} (${trend})` : `${name} reached ${show(wanted)} at sample ${hit}`); + } + if (op === "stays") { + const off = values.findIndex((v) => !same(v, wanted)); + return outcome(check, off === -1, off === -1 ? `${name} stayed ${show(wanted)}` : `${name} was ${show(values[off])} at sample ${off}`); + } + return outcome(check, false, `unknown check: probe op ${show(op)}`); +} + +export function evaluateCheck(spec: string, subject: CheckSubject): CheckOutcome { + const check = spec.trim(); + const tokens = tokenizeCheck(check); + switch (tokens[0]) { + case "exit": + return checkExit(check, tokens, subject); + case "stdout": + case "stderr": + return checkStream(check, tokens, subject); + case "status": + return checkStatus(check, tokens, subject); + case "no": + return tokens[1] === "errors" && tokens.length === 2 ? checkNoErrors(check, subject) : outcome(check, false, "unknown check"); + case "missing": + return checkMissing(check, tokens, subject); + case "probe": + return checkProbe(check, tokens, subject); + default: + return outcome(check, false, "unknown check"); + } +} + +export function evaluateChecks( + specs: readonly string[], + subject: CheckSubject, +): CheckOutcome[] { + return specs.map((spec) => evaluateCheck(spec, subject)); +} diff --git a/src/tools/verify/verify-isolation.test.ts b/src/tools/verify/verify-isolation.test.ts new file mode 100644 index 00000000..c028589e --- /dev/null +++ b/src/tools/verify/verify-isolation.test.ts @@ -0,0 +1,183 @@ +/** + * F35 — verification never writes into the deliverable. + * + * Run 13 left `verify.js`, `verification_final.txt` and + * `verification_report.txt` in the game folder: workers wrote harnesses + * next to the thing they were checking. `verify.run` takes every write + * in a throwaway copy, and `verify.syntax` picks checkers that write + * nothing (`compile()` over `py_compile`, `tsc --noEmit`, one + * `node --check` per file). These tests snapshot the workspace before + * and after and require it byte-identical. + * + * Out of scope here: `os.fs.trash` goes through Finder, which drops a + * `.DS_Store` in the directory it trashed from. That is the trash + * tool's to fix, not verification's. + */ +import { createHash } from "node:crypto"; +import { existsSync, realpathSync } from "node:fs"; +import { lstat, mkdir, mkdtemp, readdir, readFile, readlink, rm, symlink, writeFile } from "node:fs/promises"; +import net from "node:net"; +import { tmpdir } from "node:os"; +import { join, relative, resolve } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import type { AtomicAgentConfig } from "../../config/index.js"; +import { runVerify } from "./run-verify.js"; +import { verifySyntax } from "./verify-syntax.js"; +import { createVerifyWorkspace, type VerifyWorkspace } from "./verify-workspace-copy.js"; + +const NODE = process.execPath; +const REAL_TSC = realpathSync(resolve(process.cwd(), "node_modules/.bin/tsc")); +const CONFIG: Pick = { + browser: { enabled: false, channel: "chrome", headless: true, cdpUrl: null, executablePath: null, noSandbox: false, launchTimeoutMs: 1_000 }, +}; + +let work: string; + +beforeEach(async () => { + work = await mkdtemp(join(tmpdir(), "atag-verify-f35-")); + await writeFile(join(work, "index.html"), "\n"); + await mkdir(join(work, "src")); + await writeFile(join(work, "src", "app.js"), "var app = {};\n"); + await writeFile(join(work, "src", "tool.py"), "def f():\n return 1\n"); + await writeFile(join(work, "src", "run.sh"), "echo hi\n"); + await writeFile(join(work, "src", "style.css"), ".a { color: red; }\n"); + await writeFile(join(work, "src", "lib.ts"), "export const n: number = 1;\n"); + await writeFile(join(work, "tsconfig.json"), JSON.stringify({ compilerOptions: { strict: true, incremental: true, types: [] }, include: ["src/**/*.ts"] })); + await mkdir(join(work, "node_modules", ".bin"), { recursive: true }); + await symlink(REAL_TSC, join(work, "node_modules", ".bin", "tsc")); +}); + +afterEach(async () => { + await rm(work, { recursive: true, force: true }); +}); + +/** Every path under `dir` with a digest of what it is: content, link target, or "dir". */ +async function snapshotTree(dir: string, root = dir): Promise> { + const out = new Map(); + for (const entry of await readdir(dir, { withFileTypes: true })) { + const path = join(dir, entry.name); + const rel = relative(root, path); + const info = await lstat(path); + if (info.isSymbolicLink()) { + out.set(rel, `link:${await readlink(path)}`); + } else if (info.isDirectory()) { + out.set(rel, "dir"); + for (const [k, v] of await snapshotTree(path, root)) out.set(k, v); + } else { + out.set(rel, createHash("sha256").update(await readFile(path)).digest("hex")); + } + } + return out; +} + +/** + * The copies this test's runs made, so cleanup is asserted on exactly + * those directories — the shared temp dir is also where a concurrent + * test process keeps its copies. + */ +function recordingWorkspaces(): { + dirs: string[]; + workspace: (workingDir: string) => Promise; +} { + const dirs: string[] = []; + return { + dirs, + workspace: async (workingDir) => { + const ws = await createVerifyWorkspace(workingDir); + dirs.push(ws.dir); + return ws; + }, + }; +} + +function expectCopiesGone(dirs: readonly string[]): void { + expect(dirs.length).toBeGreaterThan(0); + for (const dir of dirs) { + expect(dir).not.toBe(work); + expect(existsSync(dir), `${dir} should have been removed`).toBe(false); + } +} + +async function freePort(): Promise { + return new Promise((resolve, reject) => { + const server = net.createServer(); + server.listen(0, "127.0.0.1", () => { + const address = server.address(); + const port = typeof address === "object" && address !== null ? address.port : 0; + server.close(() => resolve(port)); + }); + server.on("error", reject); + }); +} + +describe("F35 — nothing verification does reaches the workspace", () => { + it("a command that writes, appends, creates and deletes leaves the workspace byte-identical", async () => { + const before = await snapshotTree(work); + const copies = recordingWorkspaces(); + const out = await runVerify( + { + kind: "command", + cmd: NODE, + args: [ + "-e", + "const fs=require('fs');" + + "fs.writeFileSync('verify.js','harness');" + + "fs.writeFileSync('verification_report.txt','all passed');" + + "fs.mkdirSync('out'); fs.writeFileSync('out/log.txt','x');" + + "fs.appendFileSync('index.html','');" + + "fs.unlinkSync('src/app.js');" + + "process.stdout.write(fs.readdirSync('.').sort().join(','))", + ], + checks: ["exit 0", 'stdout contains "verify.js"'], + }, + { workingDir: work, config: CONFIG, workspace: copies.workspace }, + ); + // The writes happened — in the copy. + expect(out.ok).toBe(true); + expect(out.isolated).toBe(true); + expect(out.stdoutTail).toContain("verification_report.txt"); + expect(await snapshotTree(work)).toEqual(before); + expectCopiesGone(copies.dirs); + }); + + it("a service that logs to its directory leaves the workspace byte-identical", async () => { + const port = await freePort(); + const before = await snapshotTree(work); + const server = + "const http=require('http'),fs=require('fs');" + + "http.createServer((q,s)=>{ fs.appendFileSync('access.log', q.url+'\\n'); s.end('ok') }).listen(" + port + ")"; + const copies = recordingWorkspaces(); + const out = await runVerify( + { kind: "service", start: { cmd: NODE, args: ["-e", server] }, ready: { port }, requests: [{ path: "/a" }, { path: "/b" }], checks: ["status 200"] }, + { workingDir: work, config: CONFIG, workspace: copies.workspace }, + ); + expect(out.ok).toBe(true); + expect(await snapshotTree(work)).toEqual(before); + expectCopiesGone(copies.dirs); + }, 30_000); + + it("the copy is removed even when the run is killed on timeout", async () => { + const before = await snapshotTree(work); + const copies = recordingWorkspaces(); + const out = await runVerify( + { kind: "command", cmd: NODE, args: ["-e", "require('fs').writeFileSync('busy.txt','x'); setInterval(()=>{},1000)"], timeoutMs: 400 }, + { workingDir: work, config: CONFIG, workspace: copies.workspace }, + ); + expect(out.timedOut).toBe(true); + expect(await snapshotTree(work)).toEqual(before); + expectCopiesGone(copies.dirs); + }); + + it("verify.syntax leaves no bytecode, build info or temp files behind", async () => { + const before = await snapshotTree(work); + const report = await verifySyntax( + ["index.html", "src/app.js", "src/tool.py", "src/run.sh", "src/style.css", "src/lib.ts", "tsconfig.json"], + work, + ); + expect(report.failed).toBe(0); + // No `__pycache__`, no `tsconfig.tsbuildinfo` (incremental is on in + // the fixture's tsconfig on purpose), no stray temp file. + expect(await snapshotTree(work)).toEqual(before); + }, 60_000); +}); diff --git a/src/tools/verify/verify-run-args.ts b/src/tools/verify/verify-run-args.ts new file mode 100644 index 00000000..c34a2147 --- /dev/null +++ b/src/tools/verify/verify-run-args.ts @@ -0,0 +1,237 @@ +/** + * `verify.run` arguments: one shape for a command, a service or a page. + * + * Validation errors name the field, because the model reads them: a + * bare "invalid args" costs a step and teaches nothing. + */ + +export type VerifyRunKind = "command" | "service" | "page"; + +export interface VerifyRequestSpec { + readonly method?: string; + /** Relative to the service's base URL (`ready.url` origin or `127.0.0.1:port`). */ + readonly path?: string; + readonly url?: string; + readonly body?: string; + /** Default: any 2xx. */ + readonly expectStatus?: number; + /** Substring the response body must contain. */ + readonly expectBody?: string; +} + +export interface VerifyScriptStep { + readonly action: "click" | "key" | "type" | "wait"; + readonly selector?: string; + readonly key?: string; + readonly text?: string; + readonly ms?: number; +} + +export interface VerifyProbe { + readonly name: string; + /** A JavaScript expression evaluated in the page every 250 ms. */ + readonly expr: string; +} + +export interface VerifyRunArgs { + readonly kind: VerifyRunKind; + /** Relative to the throwaway copy. */ + readonly cwd?: string; + readonly env?: Readonly>; + readonly timeoutMs: number; + /** `false` (default): a soft proxy-based block, not a sandbox. */ + readonly network: boolean; + // command + readonly cmd?: string; + readonly args?: readonly string[]; + // service + readonly start?: { readonly cmd: string; readonly args?: readonly string[] }; + readonly ready?: { + readonly port?: number; + readonly url?: string; + readonly timeoutMs: number; + }; + readonly requests?: readonly VerifyRequestSpec[]; + // page + readonly path?: string; + readonly url?: string; + readonly script?: readonly VerifyScriptStep[]; + readonly seconds: number; + readonly probes?: readonly VerifyProbe[]; + // any kind + readonly checks?: readonly string[]; +} + +export const VERIFY_RUN_DEFAULT_TIMEOUT_MS = 120_000; +export const VERIFY_RUN_MAX_TIMEOUT_MS = 900_000; +export const VERIFY_READY_DEFAULT_TIMEOUT_MS = 30_000; +export const VERIFY_PAGE_DEFAULT_SECONDS = 5; +export const VERIFY_PAGE_MAX_SECONDS = 120; +const MAX_REQUESTS = 32; +const MAX_SCRIPT_STEPS = 64; +const MAX_PROBES = 16; +const MAX_CHECKS = 32; + +const KINDS: ReadonlySet = new Set(["command", "service", "page"]); +const ACTIONS: ReadonlySet = new Set(["click", "key", "type", "wait"]); + +function fail(message: string): never { + throw new Error(`verify.run: ${message}`); +} + +function optionalString(raw: unknown, field: string): string | undefined { + if (raw === undefined || raw === null) return undefined; + if (typeof raw !== "string") fail(`\`${field}\` must be a string`); + return raw; +} + +function requiredString(raw: unknown, field: string): string { + const value = optionalString(raw, field); + if (value === undefined || value.trim().length === 0) { + fail(`\`${field}\` must be a non-empty string`); + } + return value; +} + +function optionalStringArray(raw: unknown, field: string): string[] | undefined { + if (raw === undefined || raw === null) return undefined; + if (!Array.isArray(raw) || raw.some((v) => typeof v !== "string")) { + fail(`\`${field}\` must be an array of strings`); + } + return raw as string[]; +} + +function clampedNumber( + raw: unknown, + field: string, + fallback: number, + max: number, +): number { + if (raw === undefined || raw === null) return fallback; + if (typeof raw !== "number" || !Number.isFinite(raw) || raw <= 0) { + fail(`\`${field}\` must be a positive number`); + } + return Math.min(raw, max); +} + +function optionalInteger(raw: unknown, field: string): number | undefined { + if (raw === undefined || raw === null) return undefined; + if (typeof raw !== "number" || !Number.isInteger(raw)) { + fail(`\`${field}\` must be an integer`); + } + return raw; +} + +function record(raw: unknown, field: string): Record { + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) { + fail(`\`${field}\` must be an object`); + } + return raw as Record; +} + +function list(raw: unknown, field: string, max: number): unknown[] { + if (!Array.isArray(raw)) fail(`\`${field}\` must be an array`); + if (raw.length > max) fail(`\`${field}\` has ${raw.length} entries; at most ${max}`); + return raw; +} + +function parseEnv(raw: unknown): Record | undefined { + if (raw === undefined || raw === null) return undefined; + const env = record(raw, "env"); + for (const [key, value] of Object.entries(env)) { + if (typeof value !== "string") fail(`\`env.${key}\` must be a string`); + } + return env as Record; +} + +function parseRequests(raw: unknown): VerifyRequestSpec[] | undefined { + if (raw === undefined || raw === null) return undefined; + return list(raw, "requests", MAX_REQUESTS).map((entry, i) => { + const r = record(entry, `requests[${i}]`); + const path = optionalString(r.path, `requests[${i}].path`); + const url = optionalString(r.url, `requests[${i}].url`); + if ((path === undefined) === (url === undefined)) { + fail(`requests[${i}] needs exactly one of \`path\` or \`url\``); + } + return { + method: optionalString(r.method, `requests[${i}].method`), + path, + url, + body: typeof r.body === "string" || r.body === undefined || r.body === null + ? (r.body ?? undefined) + : JSON.stringify(r.body), + expectStatus: optionalInteger(r.expectStatus, `requests[${i}].expectStatus`), + expectBody: optionalString(r.expectBody, `requests[${i}].expectBody`), + }; + }); +} + +function parseScript(raw: unknown): VerifyScriptStep[] | undefined { + if (raw === undefined || raw === null) return undefined; + return list(raw, "script", MAX_SCRIPT_STEPS).map((entry, i) => { + const s = record(entry, `script[${i}]`); + const action = requiredString(s.action, `script[${i}].action`); + if (!ACTIONS.has(action)) { + fail(`script[${i}].action must be one of click, key, type, wait`); + } + const step: VerifyScriptStep = { + action: action as VerifyScriptStep["action"], + selector: optionalString(s.selector, `script[${i}].selector`), + key: optionalString(s.key, `script[${i}].key`), + text: optionalString(s.text, `script[${i}].text`), + ms: typeof s.ms === "number" ? s.ms : undefined, + }; + if (action === "click" && step.selector === undefined) fail(`script[${i}] click needs \`selector\``); + if (action === "key" && step.key === undefined) fail(`script[${i}] key needs \`key\``); + if (action === "type" && step.text === undefined) fail(`script[${i}] type needs \`text\``); + if (action === "wait" && step.ms === undefined) fail(`script[${i}] wait needs \`ms\``); + return step; + }); +} + +function parseProbes(raw: unknown): VerifyProbe[] | undefined { + if (raw === undefined || raw === null) return undefined; + return list(raw, "probes", MAX_PROBES).map((entry, i) => { + const p = record(entry, `probes[${i}]`); + return { + name: requiredString(p.name, `probes[${i}].name`), + expr: requiredString(p.expr, `probes[${i}].expr`), + }; + }); +} + +export function parseVerifyRunArgs(raw: Record): VerifyRunArgs { + const kind = requiredString(raw.kind, "kind"); + if (!KINDS.has(kind)) fail("`kind` must be one of command, service, page"); + const base = { + kind: kind as VerifyRunKind, + cwd: optionalString(raw.cwd, "cwd"), + env: parseEnv(raw.env), + timeoutMs: clampedNumber(raw.timeoutMs, "timeoutMs", VERIFY_RUN_DEFAULT_TIMEOUT_MS, VERIFY_RUN_MAX_TIMEOUT_MS), + network: raw.network === true, + seconds: clampedNumber(raw.seconds, "seconds", VERIFY_PAGE_DEFAULT_SECONDS, VERIFY_PAGE_MAX_SECONDS), + checks: raw.checks === undefined || raw.checks === null + ? undefined + : (list(raw.checks, "checks", MAX_CHECKS).map((c, i) => requiredString(c, `checks[${i}]`))), + }; + if (kind === "command") { + return { ...base, cmd: requiredString(raw.cmd, "cmd"), args: optionalStringArray(raw.args, "args") ?? [] }; + } + if (kind === "service") { + const start = record(raw.start, "start"); + const ready = raw.ready === undefined || raw.ready === null ? {} : record(raw.ready, "ready"); + const port = optionalInteger(ready.port, "ready.port"); + const url = optionalString(ready.url, "ready.url"); + if (port === undefined && url === undefined) fail("service needs `ready.port` or `ready.url`"); + return { + ...base, + start: { cmd: requiredString(start.cmd, "start.cmd"), args: optionalStringArray(start.args, "start.args") ?? [] }, + ready: { port, url, timeoutMs: clampedNumber(ready.timeoutMs, "ready.timeoutMs", VERIFY_READY_DEFAULT_TIMEOUT_MS, VERIFY_RUN_MAX_TIMEOUT_MS) }, + requests: parseRequests(raw.requests) ?? [], + }; + } + const path = optionalString(raw.path, "path"); + const url = optionalString(raw.url, "url"); + if ((path === undefined) === (url === undefined)) fail("page needs exactly one of `path` or `url`"); + return { ...base, path, url, script: parseScript(raw.script) ?? [], probes: parseProbes(raw.probes) ?? [] }; +} diff --git a/src/tools/verify/verify-run-summary.ts b/src/tools/verify/verify-run-summary.ts new file mode 100644 index 00000000..0a784aa6 --- /dev/null +++ b/src/tools/verify/verify-run-summary.ts @@ -0,0 +1,73 @@ +/** + * The `summary` of a `verify.run` result: what failed first, then what + * happened, then the output tails — clipped to 4,000 chars, because the + * whole thing lands in the prompt. + */ +import type { CheckOutcome } from "./verify-checks.js"; +import type { VerifyRunResult } from "./run-verify.js"; + +export const VERIFY_RUN_SUMMARY_MAX_CHARS = 4_000; +const LIST_LINES = 8; + +function listLines(label: string, entries: readonly string[] | undefined): string[] { + if (entries === undefined || entries.length === 0) return []; + const shown = entries.slice(0, LIST_LINES).map((e) => ` - ${e}`); + const more = entries.length > LIST_LINES ? [` … +${entries.length - LIST_LINES} more`] : []; + return [`${label} (${entries.length}):`, ...shown, ...more]; +} + +function checkLine(c: CheckOutcome): string { + return `${c.ok ? "ok" : "FAIL"} check \`${c.check}\` — ${c.detail}`; +} + +function outcomeLine(r: VerifyRunResult): string { + if (r.error !== undefined) return r.error; + if (r.kind === "command") { + const exit = r.exitCode === null ? (r.timedOut ? "killed: timed out" : "killed by signal") : `exit ${r.exitCode}`; + return `${exit} after ${r.durationMs} ms`; + } + if (r.kind === "service") { + const ready = r.ready ? `ready in ${r.readyMs} ms` : "never ready"; + const requests = r.requests ?? []; + const failed = requests.filter((q) => !q.ok).length; + return `${ready}; ${requests.length} request(s), ${failed} failed`; + } + return `page ran ${Math.round(r.durationMs / 100) / 10}s: ${r.errors?.length ?? 0} uncaught error(s), ${r.consoleErrors?.length ?? 0} console error(s), ${r.missingSelectors?.length ?? 0} failed selector lookup(s)`; +} + +function tailBlock(label: string, tail: string | undefined, budget: number): string[] { + if (tail === undefined || tail.trim().length === 0 || budget <= 40) return []; + const text = tail.length > budget ? `…${tail.slice(-(budget - 1))}` : tail; + return [`${label}:`, text]; +} + +export function renderVerifyRunSummary(r: VerifyRunResult): string { + const checks = r.checks ?? []; + const failing = checks.filter((c) => !c.ok); + const passing = checks.filter((c) => c.ok); + const head = `verify.run ${r.kind}: ${r.ok ? "ok" : "FAILED"}${r.isolated ? "" : " (ran in place — not isolated)"}`; + const lines: string[] = [head, ...failing.map(checkLine), outcomeLine(r)]; + lines.push(...listLines("uncaught errors", r.errors)); + lines.push(...listLines("console errors", r.consoleErrors)); + lines.push(...listLines("missing selectors", r.missingSelectors)); + lines.push(...listLines("failed requests", (r.requests ?? []).filter((q) => !q.ok).map((q) => `${q.method} ${q.url} → ${q.status ?? "no response"}${q.error === undefined ? "" : ` (${q.error})`}`))); + lines.push(...listLines("request failures", r.requestFailures)); + lines.push(...listLines("console warnings", r.consoleWarnings)); + if (r.probes !== undefined) { + for (const [name, samples] of Object.entries(r.probes)) { + const first = samples[0]?.[1]; + const last = samples[samples.length - 1]?.[1]; + lines.push(`probe ${name}: ${JSON.stringify(first)} → ${JSON.stringify(last)} (${samples.length} samples)`); + } + } + lines.push(...passing.map(checkLine)); + let summary = lines.join("\n"); + const remaining = VERIFY_RUN_SUMMARY_MAX_CHARS - summary.length; + const stderr = tailBlock("stderr (tail)", r.stderrTail, Math.min(1_500, remaining - 40)); + const stdout = tailBlock("stdout (tail)", r.stdoutTail, Math.min(1_500, remaining - 40 - stderr.join("\n").length)); + summary = [summary, ...stderr, ...stdout].join("\n"); + if (summary.length > VERIFY_RUN_SUMMARY_MAX_CHARS) { + summary = `${summary.slice(0, VERIFY_RUN_SUMMARY_MAX_CHARS - 12)}\n… [clipped]`; + } + return summary; +} diff --git a/src/tools/verify/verify-run.test.ts b/src/tools/verify/verify-run.test.ts new file mode 100644 index 00000000..7b540e2a --- /dev/null +++ b/src/tools/verify/verify-run.test.ts @@ -0,0 +1,119 @@ +import { mkdtemp, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +import type { ApprovalGate } from "../../approval/approval-gate.js"; +import { ApprovalDeniedError } from "../../approval/dangerous-tool.js"; +import type { AtomicAgentConfig } from "../../config/index.js"; +import type { ToolContext } from "../tool-registry.js"; +import { buildVerifyRunTool, describeVerifyRun } from "./verify-run.js"; +import { parseVerifyRunArgs } from "./verify-run-args.js"; + +const CONFIG: Pick = { + browser: { + enabled: true, + channel: "chrome", + headless: true, + cdpUrl: null, + executablePath: null, + noSandbox: false, + launchTimeoutMs: 1_000, + }, +}; + +function fakeGate(over: { approved?: boolean; scoped?: boolean } = {}): { + gate: ApprovalGate; + request: ReturnType; +} { + const request = vi.fn(async () => ({ approved: over.approved ?? true, reason: "test" })); + const gate = { + request, + fanoutScopes: { allows: () => over.scoped ?? false }, + } as unknown as ApprovalGate; + return { gate, request }; +} + +let work: string; + +beforeEach(async () => { + work = await mkdtemp(join(tmpdir(), "atag-verify-tool-")); +}); + +afterEach(async () => { + await rm(work, { recursive: true, force: true }); +}); + +function ctx(): ToolContext { + return { workingDir: work, sessionId: "s1", stepIndex: 0, signal: new AbortController().signal }; +} + +const RUN_NODE = { kind: "command", cmd: process.execPath, args: ["-e", "console.log('ran')"], checks: ["exit 0"] }; + +describe("verify.run tool", () => { + it("is read-only, asks under the shell category, and runs once approved", async () => { + const { gate, request } = fakeGate(); + const tool = buildVerifyRunTool({ approvals: gate, approvalRequired: true, config: CONFIG }); + expect(tool.readonly).toBe(true); + const out = await tool.run(RUN_NODE, ctx()); + expect(request).toHaveBeenCalledTimes(1); + expect(request.mock.calls[0]?.[0]).toMatchObject({ + sessionId: "s1", + tool: "verify.run", + category: "shell", + affectedResources: [work], + }); + expect(out.status).toBe("ok"); + expect(out.summary.split("\n")[0]).toBe("verify.run command: ok"); + expect(out.details).toMatchObject({ ok: true, kind: "command", isolated: true, exitCode: 0 }); + expect(out.details.description).toBe(`${process.execPath} -e console.log('ran')`); + }); + + it("surfaces a denial as the shared ApprovalDeniedError", async () => { + const { gate } = fakeGate({ approved: false }); + const tool = buildVerifyRunTool({ approvals: gate, approvalRequired: true, config: CONFIG }); + await expect(tool.run(RUN_NODE, ctx())).rejects.toBeInstanceOf(ApprovalDeniedError); + }); + + it("does not ask inside an authorised fan-out scope, nor with approvals off", async () => { + const scoped = fakeGate({ scoped: true }); + await buildVerifyRunTool({ approvals: scoped.gate, approvalRequired: true, config: CONFIG }).run(RUN_NODE, ctx()); + expect(scoped.request).not.toHaveBeenCalled(); + const off = fakeGate(); + await buildVerifyRunTool({ approvals: off.gate, approvalRequired: false, config: CONFIG }).run(RUN_NODE, ctx()); + expect(off.request).not.toHaveBeenCalled(); + }); + + it("answers a bad argument with an error result naming the field, before asking anyone", async () => { + const { gate, request } = fakeGate(); + const tool = buildVerifyRunTool({ approvals: gate, approvalRequired: true, config: CONFIG }); + const out = await tool.run({ kind: "service", start: { cmd: "x" } }, ctx()); + expect(out.status).toBe("error"); + expect(out.summary).toBe("verify.run: service needs `ready.port` or `ready.url`"); + expect(request).not.toHaveBeenCalled(); + }); + + it("reports a failing run as an error result with the failing checks first", async () => { + const { gate } = fakeGate(); + const tool = buildVerifyRunTool({ approvals: gate, approvalRequired: false, config: CONFIG }); + const out = await tool.run( + { kind: "command", cmd: process.execPath, args: ["-e", "process.exit(2)"], checks: ["exit 0"] }, + ctx(), + ); + expect(out.status).toBe("error"); + expect(out.summary.split("\n").slice(0, 2)).toEqual([ + "verify.run command: FAILED", + "FAIL check `exit 0` — exit 2", + ]); + }); +}); + +describe("describeVerifyRun", () => { + it("names what will run, per kind", () => { + expect(describeVerifyRun(parseVerifyRunArgs({ kind: "command", cmd: "npm", args: ["test"] }))).toBe("npm test"); + expect( + describeVerifyRun(parseVerifyRunArgs({ kind: "service", start: { cmd: "node", args: ["server.js"] }, ready: { port: 3000 }, requests: [{ path: "/" }] })), + ).toBe("start `node server.js`, then 1 request(s)"); + expect(describeVerifyRun(parseVerifyRunArgs({ kind: "page", path: "index.html", seconds: 3 }))).toBe("open index.html headless for 3s"); + }); +}); diff --git a/src/tools/verify/verify-run.ts b/src/tools/verify/verify-run.ts new file mode 100644 index 00000000..5c4e6476 --- /dev/null +++ b/src/tools/verify/verify-run.ts @@ -0,0 +1,85 @@ +/** + * `verify.run` — one tool to run what was built: a command, a service + * or a page, against a throwaway copy of the working directory. + * + * Read-only from the workspace's point of view (the copy takes every + * write), so the fusion orchestrator gate lets it run (decision D1). It + * still executes things, so below the approval ladder's shell rung it + * asks under the same category as `os.shell.run`; an authorised fan-out + * scope covers it the way it covers a worker's shell command. At level + * 5 (`--no-approval`) it runs without asking. + */ +import { requireApproval, type DangerousToolOptions } from "../../approval/dangerous-tool.js"; +import type { CompressedToolResult } from "../../compressor/result-compressor.js"; +import type { AtomicAgentConfig } from "../../config/index.js"; +import type { ToolDefinition } from "../tool-registry.js"; +import type { BrowserLauncher } from "./run-page-kind.js"; +import { runVerify } from "./run-verify.js"; +import { parseVerifyRunArgs, type VerifyRunArgs } from "./verify-run-args.js"; + +export const VERIFY_RUN_TOOL = "verify.run"; + +export interface VerifyRunToolOptions extends DangerousToolOptions { + config: Pick; + /** Test seam: replaces the real browser launch. */ + launchBrowser?: BrowserLauncher; +} + +/** What the operator is asked to approve, in one line. */ +export function describeVerifyRun(args: VerifyRunArgs): string { + if (args.kind === "command") return [args.cmd, ...(args.args ?? [])].join(" "); + if (args.kind === "service") { + const start = [args.start?.cmd, ...(args.start?.args ?? [])].join(" "); + return `start \`${start}\`, then ${args.requests?.length ?? 0} request(s)`; + } + return `open ${args.url ?? args.path} headless for ${args.seconds}s`; +} + +export function buildVerifyRunTool(options: VerifyRunToolOptions): ToolDefinition { + return { + name: VERIFY_RUN_TOOL, + description: + "Run a check against a throwaway copy of the working directory — nothing it writes reaches the workspace (may require approval). kind 'command': any test runner, compiler or script ({cmd, args}). kind 'service': {start:{cmd,args}, ready:{port|url}, requests:[{method,path|url,body,expectStatus,expectBody}]}. kind 'page': a local HTML file ({path}) or {url} in a headless browser: {script:[{action:click|key|type|wait,…}]}, then `seconds` of runtime with `probes:[{name, expr}]` sampled every 250 ms; collects uncaught errors, console errors, failed getElementById/querySelector lookups. `checks`: `exit 0`, `exit != 0`, `stdout contains \"x\"`, `stderr not contains \"x\"`, `status 200`, `no errors`, `missing selectors 0`, `probe decreases|increases|equals |reaches |stays `. `network` defaults to false (a proxy-based soft block, not a sandbox). Results are capped: output tails of 8,000 chars, summary of 4,000.", + readonly: true, + async run(rawArgs, ctx): Promise { + let args: VerifyRunArgs; + try { + args = parseVerifyRunArgs(rawArgs); + } catch (err) { + const message = err instanceof Error ? err.message : String(err); + return { tool: VERIFY_RUN_TOOL, status: "error", summary: message, details: { error: message }, truncated: false }; + } + const description = describeVerifyRun(args); + const scopedByFanout = + options.approvals.fanoutScopes?.allows(ctx.sessionId, [ctx.workingDir]) ?? false; + if (!scopedByFanout) { + await requireApproval( + options, + { + sessionId: ctx.sessionId, + tool: VERIFY_RUN_TOOL, + category: "shell", + reason: `verify (${args.kind}) in a throwaway copy of ${ctx.workingDir}`, + preview: description, + affectedResources: [ctx.workingDir], + }, + ctx.signal, + ); + } + const result = await runVerify(args, { + workingDir: ctx.workingDir, + config: options.config, + signal: ctx.signal, + ...(options.launchBrowser === undefined ? {} : { launchBrowser: options.launchBrowser }), + }); + const { summary, ...details } = result; + return { + tool: VERIFY_RUN_TOOL, + status: result.ok ? "ok" : "error", + summary, + details: { ...details, description }, + truncated: summary.endsWith("[clipped]"), + }; + }, + }; +} diff --git a/src/tools/verify/verify-syntax.test.ts b/src/tools/verify/verify-syntax.test.ts new file mode 100644 index 00000000..9568d5fb --- /dev/null +++ b/src/tools/verify/verify-syntax.test.ts @@ -0,0 +1,91 @@ +import { mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +import type { ToolContext } from "../tool-registry.js"; +import { + parseVerifySyntaxArgs, + verifySyntax, + verifySyntaxTool, +} from "./verify-syntax.js"; + +let dir: string; + +beforeEach(async () => { + dir = await mkdtemp(join(tmpdir(), "atag-verify-syntax-")); +}); + +afterEach(async () => { + await rm(dir, { recursive: true, force: true }); +}); + +function ctx(): ToolContext { + return { + workingDir: dir, + sessionId: "s1", + stepIndex: 0, + signal: new AbortController().signal, + }; +} + +describe("parseVerifySyntaxArgs", () => { + it("requires a non-empty string array and dedupes it", () => { + expect(() => parseVerifySyntaxArgs({})).toThrow(/files/); + expect(() => parseVerifySyntaxArgs({ files: [] })).toThrow(/non-empty/); + expect(() => parseVerifySyntaxArgs({ files: ["a", 1] })).toThrow(/string/); + expect(parseVerifySyntaxArgs({ files: ["a.js", "a.js", "b.js"] })).toEqual([ + "a.js", + "b.js", + ]); + }); +}); + +describe("verifySyntax", () => { + it("reports failures first, then unchecked, then passes, and never passes an unknown extension", async () => { + await writeFile(join(dir, "ok.js"), "var a = 1;\n"); + await writeFile(join(dir, "bad.json"), "{oops}"); + await writeFile(join(dir, "data.xyz"), "whatever"); + await writeFile(join(dir, "more.xyz"), "whatever"); + await writeFile(join(dir, "style.css"), ".a { color: red; }\n"); + const out = await verifySyntax( + ["ok.js", "bad.json", "data.xyz", "more.xyz", "style.css", "missing.js"], + dir, + ); + expect(out.passed).toBe(2); + expect(out.failed).toBe(2); + expect(out.unchecked).toBe(2); + const lines = out.summary.split("\n"); + expect(lines[0]).toBe( + "verify.syntax: 2 ok, 2 failed, 2 unchecked (unchecked files do not count as passing)", + ); + expect(lines[1]).toBe("no checker for .xyz (2 files)"); + expect(lines.slice(2, 4).every((l) => l.startsWith("FAIL "))).toBe(true); + expect(lines.slice(4, 6).every((l) => l.startsWith("unchecked "))).toBe(true); + expect(lines.slice(6).every((l) => l.startsWith("ok "))).toBe(true); + const byFile = new Map(out.results.map((r) => [r.file, r])); + expect(byFile.get("data.xyz")).toEqual({ + file: "data.xyz", + ok: null, + checker: "none", + error: "no checker for .xyz", + }); + expect(byFile.get("missing.js")).toMatchObject({ ok: false, error: "no such file" }); + expect(byFile.get("bad.json")?.ok).toBe(false); + }); +}); + +describe("verify.syntax tool", () => { + it("is read-only, errors when any file fails, and carries the per-file results", async () => { + expect(verifySyntaxTool.readonly).toBe(true); + await writeFile(join(dir, "a.js"), "var a = 1;\n"); + const ok = await verifySyntaxTool.run({ files: ["a.js"] }, ctx()); + expect(ok.status).toBe("ok"); + expect(ok.summary).toContain("ok a.js [node-vm]"); + await writeFile(join(dir, "b.js"), "var b = {\n"); + const failed = await verifySyntaxTool.run({ files: ["a.js", "b.js"] }, ctx()); + expect(failed.status).toBe("error"); + expect(failed.summary.split("\n")[1]).toMatch(/^FAIL b\.js \[node-vm\] — SyntaxError/); + expect(failed.details.failed).toBe(1); + }); +}); diff --git a/src/tools/verify/verify-syntax.ts b/src/tools/verify/verify-syntax.ts new file mode 100644 index 00000000..31155210 --- /dev/null +++ b/src/tools/verify/verify-syntax.ts @@ -0,0 +1,177 @@ +/** + * `verify.syntax {files}` — one checker per file, chosen by extension. + * + * Read-only, and allowed on a fusion orchestrator's turn for that reason + * (decision D1): reviewing a fan-out means looking at what came back, + * and a syntax check is looking. It writes nothing anywhere near the + * files — `node --check` is one process per file, Python is `compile()` + * without bytecode, tsc runs `--noEmit`. + * + * The one rule that matters: a file this tool did not check is reported + * as unchecked, never as passing. "No checker for .x" is a result. + */ +import { readFile, stat } from "node:fs/promises"; +import { extname } from "node:path"; + +import type { CompressedToolResult } from "../../compressor/result-compressor.js"; +import { resolveUserPath } from "../os/expand-home.js"; +import type { ToolDefinition } from "../tool-registry.js"; +import { checkCssSource } from "./check-css-syntax.js"; +import { checkHtmlSource } from "./check-html-syntax.js"; +import { + checkJavaScriptFile, + checkPythonFile, + checkShellFile, +} from "./check-script-syntax.js"; +import { checkTypeScriptFiles } from "./check-typescript-syntax.js"; +import type { SyntaxFileResult } from "./syntax-check-types.js"; + +export const VERIFY_SYNTAX_TOOL = "verify.syntax"; +export const VERIFY_SYNTAX_MAX_FILES = 200; +export const VERIFY_SUMMARY_MAX_CHARS = 4_000; + +const JS_EXTENSIONS = new Set([".js", ".cjs", ".mjs", ".json"]); +const TS_EXTENSIONS = new Set([".ts", ".tsx"]); +const SHELL_EXTENSIONS = new Set([".sh", ".bash"]); + +export interface VerifySyntaxReport { + readonly results: readonly SyntaxFileResult[]; + readonly passed: number; + readonly failed: number; + readonly unchecked: number; + readonly summary: string; +} + +export function parseVerifySyntaxArgs(raw: Record): string[] { + const files = raw.files; + if (!Array.isArray(files) || files.length === 0) { + throw new Error("verify.syntax: `files` must be a non-empty array of paths"); + } + if (files.length > VERIFY_SYNTAX_MAX_FILES) { + throw new Error( + `verify.syntax: at most ${VERIFY_SYNTAX_MAX_FILES} files per call (got ${files.length})`, + ); + } + const out: string[] = []; + for (const entry of files) { + if (typeof entry !== "string" || entry.trim().length === 0) { + throw new Error("verify.syntax: every entry of `files` must be a non-empty string"); + } + if (!out.includes(entry)) out.push(entry); + } + return out; +} + +async function checkOne( + file: string, + absolute: string, +): Promise { + const ext = extname(absolute).toLowerCase(); + if (JS_EXTENSIONS.has(ext)) { + return checkJavaScriptFile(absolute, await readFile(absolute, "utf8")).then( + (r) => ({ ...r, file }), + ); + } + if (ext === ".py") return { ...(await checkPythonFile(absolute)), file }; + if (SHELL_EXTENSIONS.has(ext)) return { ...(await checkShellFile(absolute)), file }; + if (ext === ".html" || ext === ".htm") { + return { ...(await checkHtmlSource(file, await readFile(absolute, "utf8"))) }; + } + if (ext === ".css") return checkCssSource(file, await readFile(absolute, "utf8")); + return { + file, + ok: null, + checker: "none", + error: `no checker for ${ext.length > 0 ? ext : "files without an extension"}`, + }; +} + +/** Check every file and render the report. Never throws for a bad file. */ +export async function verifySyntax( + files: readonly string[], + workingDir: string, +): Promise { + const results: SyntaxFileResult[] = new Array(files.length); + const typescript: { index: number; file: string; absolute: string }[] = []; + for (const [index, file] of files.entries()) { + const absolute = resolveUserPath(file, workingDir); + try { + const info = await stat(absolute); + if (!info.isFile()) { + results[index] = { file, ok: false, checker: "none", error: "not a regular file" }; + continue; + } + } catch { + results[index] = { file, ok: false, checker: "none", error: "no such file" }; + continue; + } + if (TS_EXTENSIONS.has(extname(absolute).toLowerCase())) { + typescript.push({ index, file, absolute }); + continue; + } + results[index] = await checkOne(file, absolute); + } + if (typescript.length > 0) { + const verdicts = await checkTypeScriptFiles(typescript); + for (const [i, entry] of typescript.entries()) { + results[entry.index] = { ...verdicts[i]!, file: entry.file }; + } + } + return report(results); +} + +function report(results: readonly SyntaxFileResult[]): VerifySyntaxReport { + const failed = results.filter((r) => r.ok === false); + const unchecked = results.filter((r) => r.ok === null); + const passed = results.filter((r) => r.ok === true); + const line = (r: SyntaxFileResult): string => { + const mark = r.ok === true ? "ok" : r.ok === false ? "FAIL" : "unchecked"; + const why = r.error === undefined ? "" : ` — ${r.error}`; + const warn = r.warning === undefined ? "" : ` ⚠ ${r.warning}`; + return `${mark} ${r.file} [${r.checker}]${why}${warn}`; + }; + const missingByExt = new Map(); + for (const r of unchecked) { + if (r.checker !== "none" || !r.error?.startsWith("no checker for ")) continue; + const ext = r.error.slice("no checker for ".length); + missingByExt.set(ext, (missingByExt.get(ext) ?? 0) + 1); + } + const head = + `verify.syntax: ${passed.length} ok, ${failed.length} failed, ${unchecked.length} unchecked` + + (unchecked.length > 0 ? " (unchecked files do not count as passing)" : ""); + const lines = [ + head, + ...[...missingByExt].map(([ext, n]) => `no checker for ${ext} (${n} file${n === 1 ? "" : "s"})`), + ...failed.map(line), + ...unchecked.map(line), + ...passed.map(line), + ]; + let summary = lines.join("\n"); + if (summary.length > VERIFY_SUMMARY_MAX_CHARS) { + summary = `${summary.slice(0, VERIFY_SUMMARY_MAX_CHARS - 12)}\n… [clipped]`; + } + return { results, passed: passed.length, failed: failed.length, unchecked: unchecked.length, summary }; +} + +export const verifySyntaxTool: ToolDefinition = { + name: VERIFY_SYNTAX_TOOL, + description: + "Syntax-check files, one checker per file by extension: .js/.cjs/.mjs/.json in-process (then node --check), .ts/.tsx via the project's tsc --noEmit, .py via python3, .sh/.bash via bash -n, .html inline