From e5cd647072cd200f058bd8bcbefece6e765b2cce Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 11 Sep 2026 21:27:12 -0700 Subject: [PATCH 1/2] Add per-director prompt size budgets assembled per family Assembled sizes drift silently as prompts grow; a numeric budget per director and model family catches bloat in CI while leaving copy edits free to land. --- src/agent/prompt-sizes.test.ts | 124 ++++++++++++++++++++++++++++ src/agent/prompt-sizes.ts | 143 +++++++++++++++++++++++++++++++++ 2 files changed, 267 insertions(+) create mode 100644 src/agent/prompt-sizes.test.ts create mode 100644 src/agent/prompt-sizes.ts diff --git a/src/agent/prompt-sizes.test.ts b/src/agent/prompt-sizes.test.ts new file mode 100644 index 000000000..495258517 --- /dev/null +++ b/src/agent/prompt-sizes.test.ts @@ -0,0 +1,124 @@ +import { describe, expect, test } from "bun:test"; +import { DIRECTOR_REGISTRY } from "./directors/registry.js"; +import { DIRECTOR_IDS, type DirectorId } from "./directors/types.js"; +import { + directorPromptSizeTable, + type PromptSizeFamily, +} from "./prompt-sizes.js"; + +/** + * Prompt size budget (CL-7664). Numeric asserts only — copy edits must not + * fail this test. Baselines were captured from the canonical fixture in + * src/agent/prompt-sizes.ts with a +2000 char / +3000 byte allowance; bytes + * get the larger headroom because multibyte copy can shift them faster. + */ +const CHAR_BUDGET: Record = { + skywalker: 28000, + builder: 49000, + explorer: 14200, + counsel: 52800, + intern: 16800, + critic: 54400, + greybeard: 54300, + neckbeard: 72300, + bruckheimer: 23400, + gaasbot: 31800, + draper: 15200, + emil: 16700, + rand: 15100, + shakespeare: 54900, + testsmith: 16200, + tester: 14000, +}; + +const BYTE_BUDGET: Record = { + skywalker: 29100, + builder: 50100, + explorer: 15300, + counsel: 53900, + intern: 17900, + critic: 55600, + greybeard: 55500, + neckbeard: 73500, + bruckheimer: 24400, + gaasbot: 32900, + draper: 16300, + emil: 17800, + rand: 16200, + shakespeare: 56000, + testsmith: 17300, + tester: 15100, +}; + +function budgetMessage( + directorId: DirectorId, + family: PromptSizeFamily, + chars: number, + bytes: number, +): string { + return ( + `Director "${directorId}" [${family}]: ${chars} chars / ${bytes} bytes ` + + `exceeds budget (${CHAR_BUDGET[directorId]} chars / ` + + `${BYTE_BUDGET[directorId]} bytes). Trim the prompt (preferred) or ` + + `consciously raise the budget here with justification. ` + + `Repro: bun -e 'import { directorPromptSizeTable, ` + + `formatPromptSizeTable } from "./src/agent/prompt-sizes.ts"; ` + + `console.log(formatPromptSizeTable(directorPromptSizeTable()))'.` + ); +} + +describe("director prompt size budget", () => { + const rows = directorPromptSizeTable(); + + test("covers every director in both families", () => { + expect(rows.length).toBe(DIRECTOR_IDS.length * 2); + for (const directorId of DIRECTOR_IDS) { + for (const family of ["default", "grok"] as const) { + expect( + rows.some((r) => r.directorId === directorId && r.family === family), + ).toBe(true); + } + } + }); + + test("every assembled prompt stays within budget", () => { + for (const row of rows) { + const overChars = row.chars > CHAR_BUDGET[row.directorId]; + const overBytes = row.bytes > BYTE_BUDGET[row.directorId]; + expect( + overChars || overBytes, + budgetMessage(row.directorId, row.family, row.chars, row.bytes), + ).toBe(false); + } + }); + + test("every assembled prompt is a real prompt, not an empty assembly", () => { + for (const row of rows) { + expect(row.chars).toBeGreaterThan(5000); + expect(row.bytes).toBeGreaterThanOrEqual(row.chars); + } + }); + + test("grok family never shrinks a prompt; only leaves grow", () => { + for (const directorId of DIRECTOR_IDS) { + const base = rows.find( + (r) => r.directorId === directorId && r.family === "default", + ); + const grok = rows.find( + (r) => r.directorId === directorId && r.family === "grok", + ); + expect(grok?.chars ?? 0).toBeGreaterThanOrEqual(base?.chars ?? 0); + if (DIRECTOR_REGISTRY[directorId].spawn.maySpawn) { + expect(grok?.chars).toBe(base?.chars); + } else { + expect(grok?.chars ?? 0).toBeGreaterThan(base?.chars ?? 0); + } + } + }); + + test("measurement is deterministic", () => { + const again = directorPromptSizeTable(); + expect(again.map((r) => r.chars)).toEqual(rows.map((r) => r.chars)); + expect(again.map((r) => r.bytes)).toEqual(rows.map((r) => r.bytes)); + }); +}); diff --git a/src/agent/prompt-sizes.ts b/src/agent/prompt-sizes.ts new file mode 100644 index 000000000..9a51cad39 --- /dev/null +++ b/src/agent/prompt-sizes.ts @@ -0,0 +1,143 @@ +import type { EnvironmentInfo } from "./environment.js"; +import { DIRECTOR_REGISTRY } from "./directors/registry.js"; +import { formatDirectorSystemPrompt } from "./directors/identity.js"; +import { + DIRECTOR_IDS, + type DirectorId, + type DirectorPackage, +} from "./directors/types.js"; +import { buildSubAgentSystemPrompt } from "./prompts.js"; +import { shouldApplyGrokAntiThrash } from "../subagent/provider-family.js"; + +/** + * Canonical prompt-size fixture (CL-7664). + * + * Assembles each director prompt exactly as src/subagent/run.ts does: + * extensions=[director systemPromptRole] + environment + tools + + * appendix, with the Grok finish-bias note gated by + * shouldApplyGrokAntiThrash (leaves on Grok-family providers only). + * + * The env and provider inputs are pinned here so sizes never drift with the + * machine, date, or checkout — only real prompt changes move the numbers. + */ +export const CANONICAL_PROMPT_ENV: EnvironmentInfo = { + cwd: "/repo", + platform: "Darwin 25.0.0", + arch: "arm64", + runtime: "Bun 1.2.0", + date: new Date("2026-01-15T12:00:00Z"), + isGitRepo: true, + gitBranch: "main", + gitDirtyCount: 0, + topLevel: "AGENTS.md CONTRIBUTING.md src/ tests/ docs/ plugins/", +}; + +const GROK_PROVIDER = { providerName: "xai/default", model: "grok-4.6" }; +const DEFAULT_PROVIDER = { + providerName: "anthropic", + model: "claude-sonnet-4", +}; + +/** Families in the size table: default assembly vs Grok (+finish-bias note). */ +export type PromptSizeFamily = "default" | "grok"; + +/** + * Canonical tool names per director, mirroring the run.ts mount order: + * package allowlist, then always-mounted manage_tasks, then leaf-only + * submit_result + ask_director, then orchestrator fleet tools + * (search_agents is Tier-1 skywalker only). + */ +export function canonicalToolNamesForDirector( + pkg: DirectorPackage, +): readonly string[] { + const names = [...(pkg.tools?.allow ?? [])]; + names.push("manage_tasks"); + if (pkg.tier === "leaf") { + names.push("submit_result", "ask_director"); + } + if (pkg.spawn.maySpawn) { + if (pkg.tier === "orchestrator") names.push("search_agents"); + names.push( + "read_agent_trace", + "spawn_agent", + "wait_agents", + "list_agents", + "close_agent", + "resume_agent", + "interrupt_agent", + "send_input", + ); + } + return names; +} + +/** Assemble one director prompt exactly as run.ts does. */ +export function assembleDirectorPrompt( + directorId: DirectorId, + family: PromptSizeFamily, +): string { + const pkg = DIRECTOR_REGISTRY[directorId]; + const orchestrator = pkg.spawn.maySpawn; + const provider = family === "grok" ? GROK_PROVIDER : DEFAULT_PROVIDER; + return buildSubAgentSystemPrompt( + [formatDirectorSystemPrompt(pkg)], + CANONICAL_PROMPT_ENV, + undefined, + { + orchestrator, + toolNames: canonicalToolNamesForDirector(pkg), + grokAntiThrash: shouldApplyGrokAntiThrash({ ...provider, orchestrator }), + }, + ); +} + +export interface DirectorPromptSize { + directorId: DirectorId; + family: PromptSizeFamily; + chars: number; + bytes: number; +} + +export function measureDirectorPrompt( + directorId: DirectorId, + family: PromptSizeFamily, +): DirectorPromptSize { + const prompt = assembleDirectorPrompt(directorId, family); + return { + directorId, + family, + chars: prompt.length, + bytes: Buffer.byteLength(prompt, "utf8"), + }; +} + +/** Full per-director x per-family size table. */ +export function directorPromptSizeTable(): DirectorPromptSize[] { + const rows: DirectorPromptSize[] = []; + for (const directorId of DIRECTOR_IDS) { + for (const family of ["default", "grok"] as const) { + rows.push(measureDirectorPrompt(directorId, family)); + } + } + return rows; +} + +/** Render the size table as markdown (for PR bodies and budget updates). */ +export function formatPromptSizeTable(rows: DirectorPromptSize[]): string { + const lines = [ + "| director | default chars (bytes) | grok chars (bytes) |", + "| --- | --- | --- |", + ]; + for (const directorId of DIRECTOR_IDS) { + const base = rows.find( + (r) => r.directorId === directorId && r.family === "default", + ); + const grok = rows.find( + (r) => r.directorId === directorId && r.family === "grok", + ); + lines.push( + `| ${directorId} | ${base?.chars} (${base?.bytes}) | ${grok?.chars} (${grok?.bytes}) |`, + ); + } + return lines.join("\n"); +} From d2546f2263cc1f91468609bffec333ea95ac9b6a Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 11 Sep 2026 21:56:43 -0700 Subject: [PATCH 2/2] Derive prompt-size tool names from the production mount path The helper re-listed allowlist names, so measured prompts exceeded production mounts: fleet verbs counted twice on orchestrators, list_dir counted though never mounted, and Codex proxies counted on non-Codex families. Build the list from the shared mount sources filtered by the same packageToCapabilities run.ts dispatches with. --- src/agent/prompt-sizes.test.ts | 105 ++++++++++++++++++++++-------- src/agent/prompt-sizes.ts | 81 ++++++++++++++++++++--- src/plugins/delete-file-plugin.ts | 2 +- 3 files changed, 153 insertions(+), 35 deletions(-) diff --git a/src/agent/prompt-sizes.test.ts b/src/agent/prompt-sizes.test.ts index 495258517..71bbf9be0 100644 --- a/src/agent/prompt-sizes.test.ts +++ b/src/agent/prompt-sizes.test.ts @@ -2,7 +2,9 @@ import { describe, expect, test } from "bun:test"; import { DIRECTOR_REGISTRY } from "./directors/registry.js"; import { DIRECTOR_IDS, type DirectorId } from "./directors/types.js"; import { + canonicalToolNamesForDirector, directorPromptSizeTable, + formatPromptSizeTable, type PromptSizeFamily, } from "./prompt-sizes.js"; @@ -13,41 +15,41 @@ import { * get the larger headroom because multibyte copy can shift them faster. */ const CHAR_BUDGET: Record = { - skywalker: 28000, - builder: 49000, + skywalker: 27000, + builder: 48800, explorer: 14200, - counsel: 52800, + counsel: 52700, intern: 16800, critic: 54400, - greybeard: 54300, + greybeard: 53600, neckbeard: 72300, - bruckheimer: 23400, - gaasbot: 31800, - draper: 15200, - emil: 16700, - rand: 15100, - shakespeare: 54900, + bruckheimer: 23200, + gaasbot: 31700, + draper: 15100, + emil: 16600, + rand: 15000, + shakespeare: 54700, testsmith: 16200, - tester: 14000, + tester: 13900, }; const BYTE_BUDGET: Record = { - skywalker: 29100, - builder: 50100, - explorer: 15300, + skywalker: 28100, + builder: 50000, + explorer: 15200, counsel: 53900, - intern: 17900, - critic: 55600, - greybeard: 55500, - neckbeard: 73500, - bruckheimer: 24400, - gaasbot: 32900, - draper: 16300, - emil: 17800, - rand: 16200, - shakespeare: 56000, - testsmith: 17300, - tester: 15100, + intern: 17800, + critic: 55500, + greybeard: 54800, + neckbeard: 73400, + bruckheimer: 24300, + gaasbot: 32800, + draper: 16200, + emil: 17700, + rand: 16100, + shakespeare: 55900, + testsmith: 17200, + tester: 15000, }; function budgetMessage( @@ -121,4 +123,55 @@ describe("director prompt size budget", () => { expect(again.map((r) => r.chars)).toEqual(rows.map((r) => r.chars)); expect(again.map((r) => r.bytes)).toEqual(rows.map((r) => r.bytes)); }); + + test("tool names match the production mount: no dupes, no phantoms", () => { + for (const directorId of DIRECTOR_IDS) { + for (const family of ["default", "grok"] as const) { + const names = canonicalToolNamesForDirector( + DIRECTOR_REGISTRY[directorId], + family, + ); + expect(new Set(names).size, `${directorId} [${family}]`).toBe( + names.length, + ); + // Neither fixture family is Codex, so the Codex proxies + // (createCodexToolProxies returns [] when !isCodex) must be absent, + // as must list_dir, which no subagent mount installs. + for (const phantom of [ + "list_dir", + "apply_patch", + "shell", + "update_plan", + ]) { + expect(names, `${directorId} [${family}]`).not.toContain(phantom); + } + if (DIRECTOR_REGISTRY[directorId].spawn.maySpawn) { + for (const verb of ["spawn_agent", "send_input"]) { + expect( + names.filter((n) => n === verb).length, + `${directorId} [${family}] mounts ${verb} once`, + ).toBe(1); + } + } + } + } + }); + + test("formatPromptSizeTable renders one row per director", () => { + const table = formatPromptSizeTable(rows); + expect(table).toContain( + "| director | default chars (bytes) | grok chars (bytes) |", + ); + for (const directorId of DIRECTOR_IDS) { + const base = rows.find( + (r) => r.directorId === directorId && r.family === "default", + ); + const grok = rows.find( + (r) => r.directorId === directorId && r.family === "grok", + ); + expect(table).toContain( + `| ${directorId} | ${base?.chars} (${base?.bytes}) | ${grok?.chars} (${grok?.bytes}) |`, + ); + } + }); }); diff --git a/src/agent/prompt-sizes.ts b/src/agent/prompt-sizes.ts index 9a51cad39..c0eaad44a 100644 --- a/src/agent/prompt-sizes.ts +++ b/src/agent/prompt-sizes.ts @@ -1,5 +1,10 @@ +import { TOOL_NAMES } from "@intx/tools-posix"; +import { LSP_TOOL_DEFINITION } from "@intx/tools-lsp"; import type { EnvironmentInfo } from "./environment.js"; -import { DIRECTOR_REGISTRY } from "./directors/registry.js"; +import { + DIRECTOR_REGISTRY, + packageToCapabilities, +} from "./directors/registry.js"; import { formatDirectorSystemPrompt } from "./directors/identity.js"; import { DIRECTOR_IDS, @@ -8,6 +13,17 @@ import { } from "./directors/types.js"; import { buildSubAgentSystemPrompt } from "./prompts.js"; import { shouldApplyGrokAntiThrash } from "../subagent/provider-family.js"; +import { isCodexProviderName } from "../config/codex-providers.js"; +import { shellCollectDefinition } from "./background-shell-tool.js"; +import { + applyPatchDefinition, + shellDefinition, + updatePlanDefinition, +} from "./codex-tool-proxies.js"; +import { manageTasksDefinition } from "./tasks.js"; +import { DELETE_FILE_DEFINITION } from "../plugins/delete-file-plugin.js"; +import { webFetchDefinition } from "../tools/web-fetch.js"; +import { webSearchDefinition } from "../tools/web-search.js"; /** * Canonical prompt-size fixture (CL-7664). @@ -42,16 +58,59 @@ const DEFAULT_PROVIDER = { export type PromptSizeFamily = "default" | "grok"; /** - * Canonical tool names per director, mirroring the run.ts mount order: - * package allowlist, then always-mounted manage_tasks, then leaf-only - * submit_result + ask_director, then orchestrator fleet tools - * (search_agents is Tier-1 skywalker only). + * Pre-filter mount names in run.ts install order: posix base (TOOL_NAMES, + * shared with createPosixTools) + delete_file / lsp plugin tools + * (buildCorePosixToolPlugins) + core web tools (coreSubAgentWebTools) + + * shell_collect (run.ts:678-694). Codex proxies (apply_patch, shell, + * update_plan) join only when isCodex — createCodexToolProxies returns [] + * otherwise (run.ts:708-718, codex-tool-proxies.ts:163-166). + */ +function preFilterMountNames(isCodex: boolean): readonly string[] { + return [ + ...Object.values(TOOL_NAMES), + DELETE_FILE_DEFINITION.name, + LSP_TOOL_DEFINITION.name, + webFetchDefinition.name, + webSearchDefinition.name, + shellCollectDefinition.name, + ...(isCodex + ? [ + applyPatchDefinition.name, + shellDefinition.name, + updatePlanDefinition.name, + ] + : []), + ]; +} + +/** + * Canonical tool names per director, assembled exactly as run.ts mounts them: + * the pre-filter set above narrowed by the package capability filter (the + * same packageToCapabilities agent-fleet dispatches with; allow keeps only + * mounted names, exclude drops denials — run.ts:720-722), then manage_tasks + * (run.ts:727-736), leaf-only submit_result + ask_director (run.ts:740-785), + * then orchestrator fleet tools with Tier-1-only search_agents + * (run.ts:791-918, tier gate at 796-797). Allowlist entries that name no + * mounted tool (list_dir, fleet verbs, off-family Codex proxies) fall out at + * the filter instead of inflating the prompt. */ export function canonicalToolNamesForDirector( pkg: DirectorPackage, + family: PromptSizeFamily, ): readonly string[] { - const names = [...(pkg.tools?.allow ?? [])]; - names.push("manage_tasks"); + const providerName = + family === "grok" + ? GROK_PROVIDER.providerName + : DEFAULT_PROVIDER.providerName; + const filtered = [...preFilterMountNames(isCodexProviderName(providerName))]; + const capabilities = packageToCapabilities(pkg); + const names = + capabilities === undefined + ? filtered + : capabilities.mode === "allow" + ? filtered.filter((name) => capabilities.tools.includes(name)) + : filtered.filter((name) => !capabilities.tools.includes(name)); + names.push(manageTasksDefinition.name); if (pkg.tier === "leaf") { names.push("submit_result", "ask_director"); } @@ -68,6 +127,12 @@ export function canonicalToolNamesForDirector( "send_input", ); } + const dupe = names.find((name, index) => names.indexOf(name) !== index); + if (dupe !== undefined) { + throw new Error( + `canonicalToolNamesForDirector(${pkg.id}): "${dupe}" mounted twice — the assembly drifted from src/subagent/run.ts`, + ); + } return names; } @@ -85,7 +150,7 @@ export function assembleDirectorPrompt( undefined, { orchestrator, - toolNames: canonicalToolNamesForDirector(pkg), + toolNames: canonicalToolNamesForDirector(pkg, family), grokAntiThrash: shouldApplyGrokAntiThrash({ ...provider, orchestrator }), }, ); diff --git a/src/plugins/delete-file-plugin.ts b/src/plugins/delete-file-plugin.ts index e5e011229..0b7905213 100644 --- a/src/plugins/delete-file-plugin.ts +++ b/src/plugins/delete-file-plugin.ts @@ -7,7 +7,7 @@ import { formatChangeDiff } from "./change-diff.js"; const DeleteFileArgs = type({ path: "string>0" }); -const DELETE_FILE_DEFINITION = { +export const DELETE_FILE_DEFINITION = { name: "delete_file", description: "Delete one file. Returns success when the file is deleted or already absent. Refuses directories; use this instead of shell rm for file deletion.",