From 67f6dae44cdd83873cbd30fb4aadf616798d817a Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 21 Aug 2026 04:10:16 -0700 Subject: [PATCH 1/3] Add tests for the adversarial output catalogue (CL-6478) Covers each seeded scenario -- malformed tool name, tool-name length boundary, textless tool-only completion, wrong-shaped and truncated tool arguments, refusal, oversized output, hallucinated tool name -- plus sequence() for scripting per-turn replies, and a regression-guard demo for CL-6478's "does the room survive the next turn?" contract. --- .../mocks/src/ollama/cl-6478-demo.test.ts | 169 +++++++++++++++ packages/mocks/src/ollama/scenarios.test.ts | 204 ++++++++++++++++++ 2 files changed, 373 insertions(+) create mode 100644 packages/mocks/src/ollama/cl-6478-demo.test.ts create mode 100644 packages/mocks/src/ollama/scenarios.test.ts diff --git a/packages/mocks/src/ollama/cl-6478-demo.test.ts b/packages/mocks/src/ollama/cl-6478-demo.test.ts new file mode 100644 index 000000000..27399a9bd --- /dev/null +++ b/packages/mocks/src/ollama/cl-6478-demo.test.ts @@ -0,0 +1,169 @@ +// Demonstrates the exact shape of test that would have caught CL-6478 +// before it shipped: qwen3.8:27b emitted `\n Promise, + body: unknown, +): Promise { + return fetchImpl( + new Request("http://mock-ollama/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ); +} + +// Stands in for hub-sessions's turn assembler persisting the PRECEDING +// assistant reply -- tool_calls included -- into the next turn's history +// before sending. This is the persistence step CL-6478's fix guards. +async function assembleAndSendNextTurn( + fetchImpl: (req: Request) => Promise, + priorAssistantMessage: unknown, + nextTurnContent: string, +): Promise { + return chat(fetchImpl, { + model: "qwen3.8:27b", + tools: [{ type: "function", function: { name: "run_shell" } }], + messages: [ + { role: "system", content: "doctrine" }, + { role: "user", content: "run the sidecar bundle" }, + priorAssistantMessage, + { role: "user", content: nextTurnContent }, + ], + stream: false, + }); +} + +describe("CL-6478 regression shape", () => { + test("a turn assembler that persists the malformed name unchanged carries it into every following turn's history", async () => { + const ollama = createOllamaMock(); + ollama.onChat( + sequence([ + ollama.reply.malformedToolName(), + ollama.reply.text("turn 2, room still alive"), + ]), + ); + + const turn1Response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + tools: [{ type: "function", function: { name: "run_shell" } }], + messages: [ + { role: "system", content: "doctrine" }, + { role: "user", content: "run the sidecar bundle" }, + ], + stream: false, + }); + const turn1Body = (await turn1Response.json()) as ChatCompletionBody; + const brokenToolCall = turn1Body.choices[0]?.message.tool_calls?.[0]; + expect(brokenToolCall?.function.name).toContain("\n + message.toolCalls?.some((call) => call.name.includes("\n { + const ollama = createOllamaMock(); + ollama.onChat( + sequence([ + ollama.reply.malformedToolName(), + ollama.reply.text("turn 2, room still alive"), + ]), + ); + + await chat(ollama.fetch, { + model: "qwen3.8:27b", + tools: [{ type: "function", function: { name: "run_shell" } }], + messages: [ + { role: "system", content: "doctrine" }, + { role: "user", content: "run the sidecar bundle" }, + ], + stream: false, + }); + + // A correct assembler runs sanitizeToolNameForPersistence (or + // equivalent) before persisting -- the malformed fragment never + // reaches history. Simulated here as the placeholder name the real + // sanitizer collapses onto (MALFORMED_TOOL_NAME in + // vendor/intx/hub-sessions/src/sanitize-tool-name.ts). + const turn2Response = await assembleAndSendNextTurn( + ollama.fetch, + { + role: "assistant", + content: null, + tool_calls: [ + { + id: "call_0", + type: "function", + function: { name: "malformed_tool_call", arguments: "{}" }, + }, + ], + }, + "did that work?", + ); + const turn2Body = (await turn2Response.json()) as ChatCompletionBody; + + expect(turn2Body.choices[0]?.message.content).toBe( + "turn 2, room still alive", + ); + expect( + ollama.requests + .last() + .messages.some((message) => + message.toolCalls?.some((call) => + call.name.includes("\n Promise, + body: unknown, +): Promise { + return fetchImpl( + new Request("http://mock-ollama/v1/chat/completions", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(body), + }), + ); +} + +type ChatCompletionBody = { + choices: { + message: { + content: string | null; + tool_calls?: { function: { name: string; arguments: string } }[]; + }; + finish_reason: string; + }[]; +}; + +describe("ollama.reply adversarial scenarios", () => { + test("malformedToolName carries CL-6478's exact leaked fragment", async () => { + const ollama = createOllamaMock(); + ollama.onChat(() => ollama.reply.malformedToolName()); + + const response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "run the sidecar bundle" }], + tools: [{ type: "function", function: { name: "run_shell" } }], + }); + const body = (await response.json()) as ChatCompletionBody; + + const name = body.choices[0]?.message.tool_calls?.[0]?.function.name; + expect(name).toContain("\n { + const ollama = createOllamaMock(); + for (const length of [63, 64, 65]) { + ollama.onChat(() => ollama.reply.toolNameOfLength(length)); + const response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "go" }], + }); + const body = (await response.json()) as ChatCompletionBody; + const name = body.choices[0]?.message.tool_calls?.[0]?.function.name; + expect(name).toHaveLength(length); + } + }); + + test("textlessToolCall carries no text, only a tool call -- a tool-only round, not an empty reply", async () => { + const ollama = createOllamaMock(); + ollama.onChat(() => + ollama.reply.textlessToolCall("create_agent", { name: "researcher" }), + ); + + const response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "make me an agent" }], + }); + const body = (await response.json()) as ChatCompletionBody; + + expect(body.choices[0]?.message.content).toBeNull(); + expect(body.choices[0]?.message.tool_calls).toHaveLength(1); + expect(body.choices[0]?.finish_reason).toBe("tool_calls"); + }); + + test("wrongShapedToolArgs is valid JSON that does not match the declared schema", async () => { + const ollama = createOllamaMock(); + ollama.onChat(() => + ollama.reply.wrongShapedToolArgs("search_flights", { + destination_typo: "SFO", + passengers: "two", + }), + ); + + const response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "book me a flight" }], + }); + const body = (await response.json()) as ChatCompletionBody; + const rawArgs = + body.choices[0]?.message.tool_calls?.[0]?.function.arguments; + + expect(() => JSON.parse(rawArgs ?? "")).not.toThrow(); + expect(JSON.parse(rawArgs ?? "{}")).toEqual({ + destination_typo: "SFO", + passengers: "two", + }); + }); + + test("truncatedToolArgs reaches the wire byte-for-byte and is NOT valid JSON", async () => { + const ollama = createOllamaMock(); + ollama.onChat(() => + ollama.reply.truncatedToolArgs("search_flights", '{"origin": "SFO"'), + ); + + const response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "book me a flight" }], + }); + const body = (await response.json()) as ChatCompletionBody; + const rawArgs = + body.choices[0]?.message.tool_calls?.[0]?.function.arguments; + + expect(rawArgs).toBe('{"origin": "SFO"'); + expect(() => JSON.parse(rawArgs ?? "")).toThrow(); + }); + + test("refusal returns plain declining text with a normal stop reason", async () => { + const ollama = createOllamaMock(); + ollama.onChat(() => ollama.reply.refusal("I can't help with that.")); + + const response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "do something unsafe" }], + }); + const body = (await response.json()) as ChatCompletionBody; + + expect(body.choices[0]?.message.content).toBe("I can't help with that."); + expect(body.choices[0]?.finish_reason).toBe("stop"); + }); + + test("oversized produces a large text blob with a length finish reason", async () => { + const ollama = createOllamaMock(); + ollama.onChat(() => ollama.reply.oversized(50_000)); + + const response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "write me a very long story" }], + }); + const body = (await response.json()) as ChatCompletionBody; + + expect(body.choices[0]?.message.content).toHaveLength(50_000); + expect(body.choices[0]?.finish_reason).toBe("length"); + }); + + test("hallucinatedToolName calls a plausible but nonexistent tool", async () => { + const ollama = createOllamaMock(); + ollama.onChat(() => ollama.reply.hallucinatedToolName()); + + const response = await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "load the git skill" }], + tools: [{ type: "function", function: { name: "load_skill" } }], + }); + const body = (await response.json()) as ChatCompletionBody; + + const name = body.choices[0]?.message.tool_calls?.[0]?.function.name; + expect(name).toBe("skills_load"); + expect(name).not.toBe("load_skill"); + }); +}); + +describe("sequence", () => { + test("scripts one reply per turn, then repeats the last one", async () => { + const ollama = createOllamaMock(); + ollama.onChat( + sequence([ollama.reply.malformedToolName(), ollama.reply.text("turn 2")]), + ); + + const turn1 = (await ( + await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "1" }], + }) + ).json()) as ChatCompletionBody; + const turn2 = (await ( + await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "2" }], + }) + ).json()) as ChatCompletionBody; + const turn3 = (await ( + await chat(ollama.fetch, { + model: "qwen3.8:27b", + messages: [{ role: "user", content: "3" }], + }) + ).json()) as ChatCompletionBody; + + expect(turn1.choices[0]?.message.tool_calls?.[0]?.function.name).toContain( + "\n { + expect(() => sequence([])).toThrow(/at least one reply/); + }); +}); From 35fb03e839e80fd093a164b32e6beab5afefe5f6 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 21 Aug 2026 04:10:26 -0700 Subject: [PATCH 2/3] Adversarial output catalogue: seeded model-misbehavior scenarios Adds named, composable ollama.reply scenarios for the specific ways real local models misbehaved last night, each documented with which model produced it and what broke: - malformedToolName() -- CL-6478's flagship case, qwen3.8:27b's \n ({ + toolCalls: [{ name: CL_6478_MALFORMED_TOOL_NAME, arguments: {} }], + finishReason: "tool_calls", + }), + toolNameOfLength: (length) => ({ + toolCalls: [{ name: nameOfExactLength(length), arguments: {} }], + finishReason: "tool_calls", + }), + textlessToolCall: (name, args = {}) => ({ + toolCalls: [{ name, arguments: args }], + finishReason: "tool_calls", + }), + wrongShapedToolArgs: (name, args) => ({ + toolCalls: [{ name, arguments: args }], + finishReason: "tool_calls", + }), + truncatedToolArgs: (name, rawArguments) => ({ + toolCalls: [{ name, arguments: undefined, rawArguments }], + finishReason: "tool_calls", + }), + refusal: (text = "I can't help with that request.") => ({ + text, + finishReason: "stop", + }), + oversized: (approxChars = 200_000) => ({ + text: "x".repeat(approxChars), + finishReason: "length", + }), + hallucinatedToolName: () => ({ + toolCalls: [{ name: HALLUCINATED_TOOL_NAME, arguments: {} }], + finishReason: "tool_calls", + }), + }; +} + +/** + * Scripts one reply per turn -- turn 1 gets `replies[0]`, turn 2 gets + * `replies[1]`, and so on; once `replies` runs out, every later turn + * repeats the last one. Pass straight to `onChat`: + * + * ```ts + * ollama.onChat(sequence([ + * ollama.reply.malformedToolName(), + * ollama.reply.text("turn 2 -- does the room survive?"), + * ])); + * ``` + * + * This is the shape CL-6478's "does the room survive the next turn?" + * contract needs: script the bad turn once, then assert the room is still + * alive on the turn after. + */ +export function sequence( + replies: readonly OllamaChatReply[], +): (request: CapturedChatRequest) => OllamaChatReply { + if (replies.length === 0) { + throw new Error("sequence() needs at least one reply"); + } + let turn = 0; + return () => { + const index = Math.min(turn, replies.length - 1); + turn += 1; + const reply = replies[index]; + if (reply === undefined) { + throw new Error("unreachable: sequence index out of bounds"); + } + return reply; + }; +} diff --git a/packages/mocks/src/ollama/types.ts b/packages/mocks/src/ollama/types.ts index 8f165d4ab..2b0e135bb 100644 --- a/packages/mocks/src/ollama/types.ts +++ b/packages/mocks/src/ollama/types.ts @@ -19,6 +19,13 @@ export type OllamaCatalogEntry = { export type OllamaToolCall = { readonly name: string; readonly arguments: unknown; + /** Wire-level override: when set, this exact string is sent as the + * `function.arguments` field instead of `JSON.stringify(arguments)` -- + * the only way to script arguments that are NOT valid JSON (a truncated + * or otherwise unparseable tool-call payload, the shape a mid-stream + * cutoff produces on a real model). Leave `arguments` as `undefined` + * when this is set. */ + readonly rawArguments?: string; }; /** From 9e124662c94d1ade8facd9d9fb8db819942f1f04 Mon Sep 17 00:00:00 2001 From: Sawyer Cutler Date: Fri, 21 Aug 2026 04:10:32 -0700 Subject: [PATCH 3/3] Update docs: document the adversarial output catalogue Moves the catalogue from the roadmap into the quickstart now that it ships, with the scenario table, the sequence() multi-turn example, and a pointer to the real CL-6478 fix in vendor/intx/hub-sessions. --- packages/mocks/README.md | 79 ++++++++++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 15 deletions(-) diff --git a/packages/mocks/README.md b/packages/mocks/README.md index 8adc36bce..78c078335 100644 --- a/packages/mocks/README.md +++ b/packages/mocks/README.md @@ -122,18 +122,68 @@ See `src/ollama/cl-6448-demo.test.ts` for the full demonstration: the same assertions failing against CL-6448's broken request shape, and passing once tools and history are actually sent. +### The adversarial output catalogue (CL-6478's shape) + +Real local models misbehave in specific, repeatable ways. Every scenario +below reproduces one we actually hit — not a hypothetical edge case — +selectable in one line straight off `ollama.reply`, exactly like `.text` +and `.toolCall`: + +```ts +ollama.onChat(() => ollama.reply.malformedToolName()); +``` + +| Scenario | What it reproduces | +| ----------------------------------- | ---------------------------------------------------------------------------------------------------------- | +| `.malformedToolName()` | CL-6478's flagship case: `qwen3.8:27b` leaked `\n Promise` handler — the in-process path. | -| `mock.listen(port?)` | Starts a real Bun HTTP server; returns `{ url, close() }` — the server path. | -| `mock.setModels(models)` | Rewrites the catalogue mid-test (a connect flow that re-reads it). | -| `mock.onChat(handler)` | Scripts every subsequent `/v1/chat/completions` reply. | -| `mock.reply.text` / `.toolCall` / `.toolCalls` | Builds an `OllamaChatReply` for a handler to return. | -| `mock.requests` | The `CapturedRequestLog` — `.all`, `.count`, `.last()`, `.clear()`. | -| `CapturedChatRequest` | One captured request: `.model`, `.tools`, `.messages`, plus every `expect*`. | +| Export | What it's for | +| -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------- | +| `createOllamaMock(options?)` | Builds one mock instance. `options.models` seeds the `/api/tags` catalogue. | +| `mock.fetch` | A `(Request) => Promise` handler — the in-process path. | +| `mock.listen(port?)` | Starts a real Bun HTTP server; returns `{ url, close() }` — the server path. | +| `mock.setModels(models)` | Rewrites the catalogue mid-test (a connect flow that re-reads it). | +| `mock.onChat(handler)` | Scripts every subsequent `/v1/chat/completions` reply. | +| `mock.reply.text` / `.toolCall` / `.toolCalls` | Builds an `OllamaChatReply` for a handler to return. | +| `mock.requests` | The `CapturedRequestLog` — `.all`, `.count`, `.last()`, `.clear()`. | +| `CapturedChatRequest` | One captured request: `.model`, `.tools`, `.messages`, plus every `expect*`. | +| `mock.reply.malformedToolName` / `.toolNameOfLength` / `.textlessToolCall` / `.wrongShapedToolArgs` / `.truncatedToolArgs` / `.refusal` / `.oversized` / `.hallucinatedToolName` | The adversarial output catalogue — see above. | +| `sequence(replies)` | Scripts one reply per turn for `onChat`; repeats the last once exhausted. | Routes covered: `GET /api/tags` (native catalogue), `POST /api/show` (per-model capability probe), `POST /v1/chat/completions` @@ -147,11 +197,10 @@ the openai adapter's request-building actually call. No `/api/generate`, - **OpenAI-compatible and Anthropic provider mocks** — `@corbits/mocks/openai` and `@corbits/mocks/anthropic`, same request-capture and reply-scripting API shape as this one. -- **Adversarial output catalogue** — seeded canned failures a handler can - return in one call: malformed tool names (`\n