Skip to content
Open
69 changes: 69 additions & 0 deletions src/agent/agent-loop.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -161,6 +161,75 @@ describe("AgentLoop end-to-end with mock LLM", () => {
expect(llmRawCompletions).toEqual([{ attempt: 1, stepIndex: 0 }]);
});

it("pins RunTurnOptions.originalRequest into the prompt once its turn is dropped (F22)", async () => {
// The record the workers' briefs quote, now reaching the
// orchestrator's own prompt: a repair turn still sees the spec.
const registry = buildDefaultToolRegistry();
const tails: string[] = [];
const loop = new AgentLoop({
registry,
slotManager: new SlotManager(2),
grammar: 'root ::= "ok"',
llmComplete: async () =>
makeCompletion(JSON.stringify({ tool: "reply", args: { text: "done" } })),
toolDescriptors: TOOLS,
capabilities: CAPS,
skillCatalog: SKILLS,
onEvent: (event) => {
if (event.type === "llm_event" && event.event.type === "prompt_captured") {
tails.push(event.event.tail);
}
},
});
const spec = `Build it: ${"detail ".repeat(30_000)}`;
const session = createEmptySessionState({ id: "s-request", workingDir });
session.turns.push({ kind: "user", text: spec, at: 1 });
session.turns.push({ kind: "assistant_reply", text: "built", at: 2 });
await loop.runTurn(session, {
userMessage: "fix these bugs",
originalRequest: spec,
maxSteps: 2,
signal: new AbortController().signal,
});
expect(tails).toHaveLength(1);
expect(tails[0]).toContain("### request");
expect(tails[0]!.indexOf("### request")).toBeLessThan(tails[0]!.indexOf("### conversation"));
});

it("passes RunTurnOptions.reasoningEffort / maxOutputTokens to every completion (F20)", async () => {
const registry = buildDefaultToolRegistry();
const seen: Array<{ effort?: string; cap?: number }> = [];
const loop = new AgentLoop({
registry,
slotManager: new SlotManager(2),
grammar: 'root ::= "ok"',
llmComplete: async (params) => {
seen.push({
...(params.reasoningEffort === undefined ? {} : { effort: params.reasoningEffort }),
...(params.maxOutputTokens === undefined ? {} : { cap: params.maxOutputTokens }),
});
return makeCompletion(JSON.stringify({ tool: "reply", args: { text: "done" } }));
},
toolDescriptors: TOOLS,
capabilities: CAPS,
skillCatalog: SKILLS,
});
const session = createEmptySessionState({ id: "s-effort", workingDir });
await loop.runTurn(session, {
userMessage: "go",
reasoningEffort: "low",
maxOutputTokens: 12_000,
maxSteps: 2,
signal: new AbortController().signal,
});
await loop.runTurn(session, {
userMessage: "again",
maxSteps: 2,
signal: new AbortController().signal,
});
expect(seen).toEqual([{ effort: "low", cap: 12_000 }, {}]);
});

it("keeps working past the leg length while the task is progressing", async () => {
// The point of the change: `maxSteps` is a checkpoint, not the end
// of the work. A task that is still getting usable results out of
Expand Down
57 changes: 56 additions & 1 deletion src/agent/agent-loop.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@ import type {
StreamChunk,
} from "../llm/llama-server-client.js";
import type { SlotManager } from "../llm/slot-manager.js";
import type { ToolCallTransport } from "../llm/provider/completion-types.js";
import type {
ReasoningEffort,
ToolCallTransport,
} from "../llm/provider/completion-types.js";
import type { ToolCallAdapter } from "../llm/provider/adapters/tool-call-adapter.js";
import {
PLAIN_INSTRUCT_PROFILE,
Expand Down Expand Up @@ -142,6 +145,15 @@ export interface AgentLoopDependencies {
* reflected without restarting the loop.
*/
contextWindow?: () => number | null;
/**
* The local worker leg's request-slot count as the server reported it
* (`SlotManager.observedPoolSize`), `null` until a `/props` answer has
* sized the pool. Read per step; it reaches the `### fusion` machine
* facts for an external llama-server whose `--parallel` the config
* cannot state. Moves once — when the pool is first observed — and the
* prefix moves with it, the same cost as a config write.
*/
liveWorkerSlots?: () => number | null;
/**
* The model server just revealed its real context window: a reply
* stopped `context_window`-truncated after this many prompt + reply
Expand Down Expand Up @@ -520,6 +532,26 @@ export interface RunTurnOptions {
signal: AbortSignal;
/** Optional new user message to append before stepping. */
userMessage?: string;
/**
* The operator's request behind this turn, as the runtime records it
* for the workers' briefs (`pickOriginalRequest`). Reaches every step's
* prompt as `### request` once the packer has dropped the user turn
* that carried it, so a repair turn still sees the spec. Absent in
* test / legacy wiring, where nothing is pinned.
*/
originalRequest?: string;
/**
* Reasoning effort for every completion of this turn, mapped per
* provider family by the body builder. A fusion worker's
* `workerReasoning`; absent, the provider's default.
*/
reasoningEffort?: ReasoningEffort;
/**
* Output ceiling for every completion of this turn, below the
* provider's own. A fusion worker's `workerMaxOutputTokens`; the
* truncation retry's per-step cap still wins over it.
*/
maxOutputTokens?: number;
/**
* Pin every completion of this turn to one configured provider id.
* The step is built for that link's transport (via
Expand Down Expand Up @@ -959,6 +991,16 @@ export class AgentLoop {
// the per-request grammar — see `tool-roles.ts`.
const toolRole: ToolRole =
options.toolRole ?? (fusionOrchestratorTurn ? "orchestrator" : "full");
// Claims need evidence, once per turn: a reply that reports a check
// nothing ran is held back and noticed the first time only
// (`claim-evidence.ts`); the second is delivered and marked.
let claimNoticeGiven = false;
const claimEvidence = {
noticed: () => claimNoticeGiven,
markNoticed: () => {
claimNoticeGiven = true;
},
};

let reason: AgentLoopReason = "max_steps";
let stepsTaken = 0;
Expand Down Expand Up @@ -1279,6 +1321,15 @@ export class AgentLoop {
...(options.userMessage !== undefined
? { userMessage: options.userMessage }
: {}),
...(options.originalRequest !== undefined
? { originalRequest: options.originalRequest }
: {}),
...(options.reasoningEffort !== undefined
? { reasoningEffort: options.reasoningEffort }
: {}),
...(options.maxOutputTokens !== undefined
? { maxOutputTokens: options.maxOutputTokens }
: {}),
},
{
registry: this.deps.registry,
Expand All @@ -1297,12 +1348,16 @@ export class AgentLoop {
},
}
: {}),
claimEvidence,
slotManager: this.deps.slotManager,
grammar: activeGrammar,
profile: activeProfile,
...(this.deps.contextWindow
? { contextWindow: this.deps.contextWindow() }
: {}),
...(this.deps.liveWorkerSlots
? { liveWorkerSlots: this.deps.liveWorkerSlots }
: {}),
toolTransport:
pinnedSlice?.toolTransport ??
this.deps.toolTransport ??
Expand Down
98 changes: 98 additions & 0 deletions src/agent/claim-evidence.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
import { describe, expect, it } from "vitest";
import type { ConversationTurn } from "../session/conversation-turn.js";
import {
claimHasEvidence,
detectCheckClaims,
formatUnverifiedClaimNotice,
turnToolCalls,
unverifiedClaims,
} from "./claim-evidence.js";

const shell = (cmd: string, args: string[] = []) => ({
tool: "os.shell.run",
args: { cmd, args },
});

describe("detectCheckClaims", () => {
it("finds the claims the failing replies made, one per kind, in order", () => {
// Run 12: "Ran node --check on all JavaScript files (all passed)".
expect(
detectCheckClaims("Ran node --check on all JavaScript files (all passed)."),
).toEqual([{ kind: "node-check", text: "node --check" }]);
expect(detectCheckClaims("All 12 tests pass. I verified the layout.")).toEqual([
{ kind: "tests", text: "tests pass" },
{ kind: "verified", text: "verified" },
]);
expect(detectCheckClaims("I ran the tests and lint passes; it builds cleanly.")).toEqual([
{ kind: "tests", text: "ran the tests" },
{ kind: "lint", text: "lint passes" },
{ kind: "build", text: "builds cleanly" },
]);
});

it("does not fire on ordinary prose", () => {
expect(detectCheckClaims("Here is the file. Run the tests when you like.")).toEqual([]);
expect(detectCheckClaims("The build step is next; nothing checked yet.")).toEqual([]);
});
});

describe("claimHasEvidence", () => {
it("accepts a shell command containing the claimed check", () => {
const nodeCheck = { kind: "node-check" as const, text: "node --check" };
expect(claimHasEvidence(nodeCheck, [shell("node", ["--check", "a.js"])])).toBe(true);
expect(claimHasEvidence(nodeCheck, [shell("node -c js/a.js")])).toBe(true);
expect(claimHasEvidence(nodeCheck, [shell("node", ["js/a.js"])])).toBe(false);
expect(claimHasEvidence(nodeCheck, [shell("ls")])).toBe(false);
const tests = { kind: "tests" as const, text: "tests pass" };
expect(claimHasEvidence(tests, [shell("npx", ["vitest", "run"])])).toBe(true);
expect(claimHasEvidence(tests, [shell("npm test")])).toBe(true);
expect(claimHasEvidence(tests, [shell("cat", ["a.js"])])).toBe(false);
});

it("accepts any verify.* call for any claim", () => {
for (const kind of ["node-check", "tests", "verified", "lint", "build"] as const) {
expect(
claimHasEvidence({ kind, text: "x" }, [{ tool: "verify.syntax", args: {} }]),
).toBe(true);
}
});

it("reads a bare 'verified' as backed by any command the turn ran", () => {
const verified = { kind: "verified" as const, text: "verified" };
expect(claimHasEvidence(verified, [shell("ls")])).toBe(true);
expect(claimHasEvidence(verified, [{ tool: "os.fs.read", args: {} }])).toBe(false);
});
});

describe("unverifiedClaims / turnToolCalls", () => {
it("returns only the claims nothing backs", () => {
expect(
unverifiedClaims("tests pass and node --check passed", [shell("node --check a.js")]),
).toEqual([{ kind: "tests", text: "tests pass" }]);
expect(unverifiedClaims("tests pass", [{ tool: "verify.run", args: {} }])).toEqual([]);
});

it("reads this turn's calls: everything after the last user turn", () => {
const turns: ConversationTurn[] = [
{ kind: "user", text: "build it", at: 1 },
{ kind: "assistant_tool_call", tool: "os.shell.run", args: { cmd: "node --check a.js" }, at: 2 },
{ kind: "tool_result", tool: "os.shell.run", status: "ok", summary: "", at: 3 },
{ kind: "assistant_reply", text: "done", at: 4 },
{ kind: "user", text: "now fix it", at: 5 },
{ kind: "assistant_tool_call", tool: "os.fs.read", args: { path: "a.js" }, at: 6 },
{ kind: "tool_result", tool: "os.fs.read", status: "ok", summary: "", at: 7 },
];
expect(turnToolCalls(turns)).toEqual([{ tool: "os.fs.read", args: { path: "a.js" } }]);
// The earlier turn's node --check is not evidence for this turn.
expect(unverifiedClaims("node --check passed", turnToolCalls(turns))).toHaveLength(1);
});

it("names the claim and both exits in the notice", () => {
const notice = formatUnverifiedClaimNotice([{ kind: "tests", text: "tests pass" }]);
expect(notice).toContain('claims "tests pass"');
expect(notice).toContain("no such check ran this turn");
expect(notice).toContain("`verify.run`");
expect(notice).toContain("`os.shell.run`");
expect(notice).toContain("remove the claim");
});
});
Loading