diff --git a/src/agent/prompt-sizes.test.ts b/src/agent/prompt-sizes.test.ts new file mode 100644 index 000000000..71bbf9be0 --- /dev/null +++ b/src/agent/prompt-sizes.test.ts @@ -0,0 +1,177 @@ +import { describe, expect, test } from "bun:test"; +import { DIRECTOR_REGISTRY } from "./directors/registry.js"; +import { DIRECTOR_IDS, type DirectorId } from "./directors/types.js"; +import { + canonicalToolNamesForDirector, + directorPromptSizeTable, + formatPromptSizeTable, + type PromptSizeFamily, +} from "./prompt-sizes.js"; + +/** + * Prompt size budget (CL-7664). Numeric asserts only — copy edits must not + * fail this test. Baselines were captured from the canonical fixture in + * src/agent/prompt-sizes.ts with a +2000 char / +3000 byte allowance; bytes + * get the larger headroom because multibyte copy can shift them faster. + */ +const CHAR_BUDGET: Record = { + skywalker: 27000, + builder: 48800, + explorer: 14200, + counsel: 52700, + intern: 16800, + critic: 54400, + greybeard: 53600, + neckbeard: 72300, + bruckheimer: 23200, + gaasbot: 31700, + draper: 15100, + emil: 16600, + rand: 15000, + shakespeare: 54700, + testsmith: 16200, + tester: 13900, +}; + +const BYTE_BUDGET: Record = { + skywalker: 28100, + builder: 50000, + explorer: 15200, + counsel: 53900, + intern: 17800, + critic: 55500, + greybeard: 54800, + neckbeard: 73400, + bruckheimer: 24300, + gaasbot: 32800, + draper: 16200, + emil: 17700, + rand: 16100, + shakespeare: 55900, + testsmith: 17200, + tester: 15000, +}; + +function budgetMessage( + directorId: DirectorId, + family: PromptSizeFamily, + chars: number, + bytes: number, +): string { + return ( + `Director "${directorId}" [${family}]: ${chars} chars / ${bytes} bytes ` + + `exceeds budget (${CHAR_BUDGET[directorId]} chars / ` + + `${BYTE_BUDGET[directorId]} bytes). Trim the prompt (preferred) or ` + + `consciously raise the budget here with justification. ` + + `Repro: bun -e 'import { directorPromptSizeTable, ` + + `formatPromptSizeTable } from "./src/agent/prompt-sizes.ts"; ` + + `console.log(formatPromptSizeTable(directorPromptSizeTable()))'.` + ); +} + +describe("director prompt size budget", () => { + const rows = directorPromptSizeTable(); + + test("covers every director in both families", () => { + expect(rows.length).toBe(DIRECTOR_IDS.length * 2); + for (const directorId of DIRECTOR_IDS) { + for (const family of ["default", "grok"] as const) { + expect( + rows.some((r) => r.directorId === directorId && r.family === family), + ).toBe(true); + } + } + }); + + test("every assembled prompt stays within budget", () => { + for (const row of rows) { + const overChars = row.chars > CHAR_BUDGET[row.directorId]; + const overBytes = row.bytes > BYTE_BUDGET[row.directorId]; + expect( + overChars || overBytes, + budgetMessage(row.directorId, row.family, row.chars, row.bytes), + ).toBe(false); + } + }); + + test("every assembled prompt is a real prompt, not an empty assembly", () => { + for (const row of rows) { + expect(row.chars).toBeGreaterThan(5000); + expect(row.bytes).toBeGreaterThanOrEqual(row.chars); + } + }); + + test("grok family never shrinks a prompt; only leaves grow", () => { + for (const directorId of DIRECTOR_IDS) { + const base = rows.find( + (r) => r.directorId === directorId && r.family === "default", + ); + const grok = rows.find( + (r) => r.directorId === directorId && r.family === "grok", + ); + expect(grok?.chars ?? 0).toBeGreaterThanOrEqual(base?.chars ?? 0); + if (DIRECTOR_REGISTRY[directorId].spawn.maySpawn) { + expect(grok?.chars).toBe(base?.chars); + } else { + expect(grok?.chars ?? 0).toBeGreaterThan(base?.chars ?? 0); + } + } + }); + + test("measurement is deterministic", () => { + const again = directorPromptSizeTable(); + expect(again.map((r) => r.chars)).toEqual(rows.map((r) => r.chars)); + expect(again.map((r) => r.bytes)).toEqual(rows.map((r) => r.bytes)); + }); + + test("tool names match the production mount: no dupes, no phantoms", () => { + for (const directorId of DIRECTOR_IDS) { + for (const family of ["default", "grok"] as const) { + const names = canonicalToolNamesForDirector( + DIRECTOR_REGISTRY[directorId], + family, + ); + expect(new Set(names).size, `${directorId} [${family}]`).toBe( + names.length, + ); + // Neither fixture family is Codex, so the Codex proxies + // (createCodexToolProxies returns [] when !isCodex) must be absent, + // as must list_dir, which no subagent mount installs. + for (const phantom of [ + "list_dir", + "apply_patch", + "shell", + "update_plan", + ]) { + expect(names, `${directorId} [${family}]`).not.toContain(phantom); + } + if (DIRECTOR_REGISTRY[directorId].spawn.maySpawn) { + for (const verb of ["spawn_agent", "send_input"]) { + expect( + names.filter((n) => n === verb).length, + `${directorId} [${family}] mounts ${verb} once`, + ).toBe(1); + } + } + } + } + }); + + test("formatPromptSizeTable renders one row per director", () => { + const table = formatPromptSizeTable(rows); + expect(table).toContain( + "| director | default chars (bytes) | grok chars (bytes) |", + ); + for (const directorId of DIRECTOR_IDS) { + const base = rows.find( + (r) => r.directorId === directorId && r.family === "default", + ); + const grok = rows.find( + (r) => r.directorId === directorId && r.family === "grok", + ); + expect(table).toContain( + `| ${directorId} | ${base?.chars} (${base?.bytes}) | ${grok?.chars} (${grok?.bytes}) |`, + ); + } + }); +}); diff --git a/src/agent/prompt-sizes.ts b/src/agent/prompt-sizes.ts new file mode 100644 index 000000000..c0eaad44a --- /dev/null +++ b/src/agent/prompt-sizes.ts @@ -0,0 +1,208 @@ +import { TOOL_NAMES } from "@intx/tools-posix"; +import { LSP_TOOL_DEFINITION } from "@intx/tools-lsp"; +import type { EnvironmentInfo } from "./environment.js"; +import { + DIRECTOR_REGISTRY, + packageToCapabilities, +} from "./directors/registry.js"; +import { formatDirectorSystemPrompt } from "./directors/identity.js"; +import { + DIRECTOR_IDS, + type DirectorId, + type DirectorPackage, +} from "./directors/types.js"; +import { buildSubAgentSystemPrompt } from "./prompts.js"; +import { shouldApplyGrokAntiThrash } from "../subagent/provider-family.js"; +import { isCodexProviderName } from "../config/codex-providers.js"; +import { shellCollectDefinition } from "./background-shell-tool.js"; +import { + applyPatchDefinition, + shellDefinition, + updatePlanDefinition, +} from "./codex-tool-proxies.js"; +import { manageTasksDefinition } from "./tasks.js"; +import { DELETE_FILE_DEFINITION } from "../plugins/delete-file-plugin.js"; +import { webFetchDefinition } from "../tools/web-fetch.js"; +import { webSearchDefinition } from "../tools/web-search.js"; + +/** + * Canonical prompt-size fixture (CL-7664). + * + * Assembles each director prompt exactly as src/subagent/run.ts does: + * extensions=[director systemPromptRole] + environment + tools + + * appendix, with the Grok finish-bias note gated by + * shouldApplyGrokAntiThrash (leaves on Grok-family providers only). + * + * The env and provider inputs are pinned here so sizes never drift with the + * machine, date, or checkout — only real prompt changes move the numbers. + */ +export const CANONICAL_PROMPT_ENV: EnvironmentInfo = { + cwd: "/repo", + platform: "Darwin 25.0.0", + arch: "arm64", + runtime: "Bun 1.2.0", + date: new Date("2026-01-15T12:00:00Z"), + isGitRepo: true, + gitBranch: "main", + gitDirtyCount: 0, + topLevel: "AGENTS.md CONTRIBUTING.md src/ tests/ docs/ plugins/", +}; + +const GROK_PROVIDER = { providerName: "xai/default", model: "grok-4.6" }; +const DEFAULT_PROVIDER = { + providerName: "anthropic", + model: "claude-sonnet-4", +}; + +/** Families in the size table: default assembly vs Grok (+finish-bias note). */ +export type PromptSizeFamily = "default" | "grok"; + +/** + * Pre-filter mount names in run.ts install order: posix base (TOOL_NAMES, + * shared with createPosixTools) + delete_file / lsp plugin tools + * (buildCorePosixToolPlugins) + core web tools (coreSubAgentWebTools) + + * shell_collect (run.ts:678-694). Codex proxies (apply_patch, shell, + * update_plan) join only when isCodex — createCodexToolProxies returns [] + * otherwise (run.ts:708-718, codex-tool-proxies.ts:163-166). + */ +function preFilterMountNames(isCodex: boolean): readonly string[] { + return [ + ...Object.values(TOOL_NAMES), + DELETE_FILE_DEFINITION.name, + LSP_TOOL_DEFINITION.name, + webFetchDefinition.name, + webSearchDefinition.name, + shellCollectDefinition.name, + ...(isCodex + ? [ + applyPatchDefinition.name, + shellDefinition.name, + updatePlanDefinition.name, + ] + : []), + ]; +} + +/** + * Canonical tool names per director, assembled exactly as run.ts mounts them: + * the pre-filter set above narrowed by the package capability filter (the + * same packageToCapabilities agent-fleet dispatches with; allow keeps only + * mounted names, exclude drops denials — run.ts:720-722), then manage_tasks + * (run.ts:727-736), leaf-only submit_result + ask_director (run.ts:740-785), + * then orchestrator fleet tools with Tier-1-only search_agents + * (run.ts:791-918, tier gate at 796-797). Allowlist entries that name no + * mounted tool (list_dir, fleet verbs, off-family Codex proxies) fall out at + * the filter instead of inflating the prompt. + */ +export function canonicalToolNamesForDirector( + pkg: DirectorPackage, + family: PromptSizeFamily, +): readonly string[] { + const providerName = + family === "grok" + ? GROK_PROVIDER.providerName + : DEFAULT_PROVIDER.providerName; + const filtered = [...preFilterMountNames(isCodexProviderName(providerName))]; + const capabilities = packageToCapabilities(pkg); + const names = + capabilities === undefined + ? filtered + : capabilities.mode === "allow" + ? filtered.filter((name) => capabilities.tools.includes(name)) + : filtered.filter((name) => !capabilities.tools.includes(name)); + names.push(manageTasksDefinition.name); + if (pkg.tier === "leaf") { + names.push("submit_result", "ask_director"); + } + if (pkg.spawn.maySpawn) { + if (pkg.tier === "orchestrator") names.push("search_agents"); + names.push( + "read_agent_trace", + "spawn_agent", + "wait_agents", + "list_agents", + "close_agent", + "resume_agent", + "interrupt_agent", + "send_input", + ); + } + const dupe = names.find((name, index) => names.indexOf(name) !== index); + if (dupe !== undefined) { + throw new Error( + `canonicalToolNamesForDirector(${pkg.id}): "${dupe}" mounted twice — the assembly drifted from src/subagent/run.ts`, + ); + } + return names; +} + +/** Assemble one director prompt exactly as run.ts does. */ +export function assembleDirectorPrompt( + directorId: DirectorId, + family: PromptSizeFamily, +): string { + const pkg = DIRECTOR_REGISTRY[directorId]; + const orchestrator = pkg.spawn.maySpawn; + const provider = family === "grok" ? GROK_PROVIDER : DEFAULT_PROVIDER; + return buildSubAgentSystemPrompt( + [formatDirectorSystemPrompt(pkg)], + CANONICAL_PROMPT_ENV, + undefined, + { + orchestrator, + toolNames: canonicalToolNamesForDirector(pkg, family), + grokAntiThrash: shouldApplyGrokAntiThrash({ ...provider, orchestrator }), + }, + ); +} + +export interface DirectorPromptSize { + directorId: DirectorId; + family: PromptSizeFamily; + chars: number; + bytes: number; +} + +export function measureDirectorPrompt( + directorId: DirectorId, + family: PromptSizeFamily, +): DirectorPromptSize { + const prompt = assembleDirectorPrompt(directorId, family); + return { + directorId, + family, + chars: prompt.length, + bytes: Buffer.byteLength(prompt, "utf8"), + }; +} + +/** Full per-director x per-family size table. */ +export function directorPromptSizeTable(): DirectorPromptSize[] { + const rows: DirectorPromptSize[] = []; + for (const directorId of DIRECTOR_IDS) { + for (const family of ["default", "grok"] as const) { + rows.push(measureDirectorPrompt(directorId, family)); + } + } + return rows; +} + +/** Render the size table as markdown (for PR bodies and budget updates). */ +export function formatPromptSizeTable(rows: DirectorPromptSize[]): string { + const lines = [ + "| director | default chars (bytes) | grok chars (bytes) |", + "| --- | --- | --- |", + ]; + for (const directorId of DIRECTOR_IDS) { + const base = rows.find( + (r) => r.directorId === directorId && r.family === "default", + ); + const grok = rows.find( + (r) => r.directorId === directorId && r.family === "grok", + ); + lines.push( + `| ${directorId} | ${base?.chars} (${base?.bytes}) | ${grok?.chars} (${grok?.bytes}) |`, + ); + } + return lines.join("\n"); +} diff --git a/src/plugins/delete-file-plugin.ts b/src/plugins/delete-file-plugin.ts index e5e011229..0b7905213 100644 --- a/src/plugins/delete-file-plugin.ts +++ b/src/plugins/delete-file-plugin.ts @@ -7,7 +7,7 @@ import { formatChangeDiff } from "./change-diff.js"; const DeleteFileArgs = type({ path: "string>0" }); -const DELETE_FILE_DEFINITION = { +export const DELETE_FILE_DEFINITION = { name: "delete_file", description: "Delete one file. Returns success when the file is deleted or already absent. Refuses directories; use this instead of shell rm for file deletion.",