Skip to content

Commit 9bfa3a7

Browse files
TheGreatAxiosTheGreatAxios
committed
feat(evals): add task-completion baseline harness for CL-7932
Summary: repeatable harness runs a frozen 4-task set end to end through the production agent loop with scripted mock responders, reporting completion rate plus control-layer signals (turns, retries, compaction events, doom-loop/thrash interventions, wall clock). Baseline on b509ed3: 50.0% (4/8 runs, 2 repeats), deterministic across repeats. Task set frozen as tasks.json v1 for the post-0.4.x re-measure. Verification: bun scripts/eval-completion.ts --repeats 2 --out evals/completion/baseline-2026-09-14.json; bun test evals/completion/lib.test.ts (7 pass); bun run check (exit 0).
1 parent 174d7e3 commit 9bfa3a7

17 files changed

Lines changed: 1087 additions & 0 deletions

File tree

‎evals/completion/baseline-2026-09-14.json‎

Lines changed: 188 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,188 @@
1+
{
2+
"harness": "completion-baseline",
3+
"version": 1,
4+
"startedAt": "2026-09-14T07:30:17.369Z",
5+
"finishedAt": "2026-09-14T07:30:19.083Z",
6+
"commitSha": "b509ed3edb0db5368eb190528d3beaee8895837b",
7+
"provider": "stub-scripted",
8+
"model": "completion-baseline-v1",
9+
"repeats": 2,
10+
"taskSetVersion": 1,
11+
"taskIds": ["version-endpoint", "sum-fix", "decline-refactor", "stall-read"],
12+
"results": [
13+
{
14+
"taskId": "version-endpoint",
15+
"title": "Add GET /version to a two-route service",
16+
"profile": "solve",
17+
"repeat": 0,
18+
"completed": true,
19+
"runStatus": "completed",
20+
"turnsUsed": 3,
21+
"toolCallCount": 2,
22+
"failedToolCalls": 0,
23+
"retryCount": 0,
24+
"compactionEvents": 0,
25+
"doomLoopInterventions": 0,
26+
"thrashInterventions": 0,
27+
"gateSuspensions": 0,
28+
"agentDurationMs": 163,
29+
"verifyDurationMs": 149,
30+
"verifyExitCode": 0,
31+
"overBudget": false
32+
},
33+
{
34+
"taskId": "version-endpoint",
35+
"title": "Add GET /version to a two-route service",
36+
"profile": "solve",
37+
"repeat": 1,
38+
"completed": true,
39+
"runStatus": "completed",
40+
"turnsUsed": 3,
41+
"toolCallCount": 2,
42+
"failedToolCalls": 0,
43+
"retryCount": 0,
44+
"compactionEvents": 0,
45+
"doomLoopInterventions": 0,
46+
"thrashInterventions": 0,
47+
"gateSuspensions": 0,
48+
"agentDurationMs": 62,
49+
"verifyDurationMs": 141,
50+
"verifyExitCode": 0,
51+
"overBudget": false
52+
},
53+
{
54+
"taskId": "sum-fix",
55+
"title": "Fix an off-by-one in a total function",
56+
"profile": "solve",
57+
"repeat": 0,
58+
"completed": true,
59+
"runStatus": "completed",
60+
"turnsUsed": 3,
61+
"toolCallCount": 2,
62+
"failedToolCalls": 0,
63+
"retryCount": 0,
64+
"compactionEvents": 0,
65+
"doomLoopInterventions": 0,
66+
"thrashInterventions": 0,
67+
"gateSuspensions": 0,
68+
"agentDurationMs": 131,
69+
"verifyDurationMs": 135,
70+
"verifyExitCode": 0,
71+
"overBudget": false
72+
},
73+
{
74+
"taskId": "sum-fix",
75+
"title": "Fix an off-by-one in a total function",
76+
"profile": "solve",
77+
"repeat": 1,
78+
"completed": true,
79+
"runStatus": "completed",
80+
"turnsUsed": 3,
81+
"toolCallCount": 2,
82+
"failedToolCalls": 0,
83+
"retryCount": 0,
84+
"compactionEvents": 0,
85+
"doomLoopInterventions": 0,
86+
"thrashInterventions": 0,
87+
"gateSuspensions": 0,
88+
"agentDurationMs": 66,
89+
"verifyDurationMs": 127,
90+
"verifyExitCode": 0,
91+
"overBudget": false
92+
},
93+
{
94+
"taskId": "decline-refactor",
95+
"title": "Decline a migration that does not fit the turn budget",
96+
"profile": "decline",
97+
"repeat": 0,
98+
"completed": false,
99+
"runStatus": "completed",
100+
"turnsUsed": 1,
101+
"toolCallCount": 0,
102+
"failedToolCalls": 0,
103+
"retryCount": 0,
104+
"compactionEvents": 0,
105+
"doomLoopInterventions": 0,
106+
"thrashInterventions": 0,
107+
"gateSuspensions": 0,
108+
"agentDurationMs": 30,
109+
"verifyDurationMs": 72,
110+
"verifyExitCode": 1,
111+
"overBudget": false
112+
},
113+
{
114+
"taskId": "decline-refactor",
115+
"title": "Decline a migration that does not fit the turn budget",
116+
"profile": "decline",
117+
"repeat": 1,
118+
"completed": false,
119+
"runStatus": "completed",
120+
"turnsUsed": 1,
121+
"toolCallCount": 0,
122+
"failedToolCalls": 0,
123+
"retryCount": 0,
124+
"compactionEvents": 0,
125+
"doomLoopInterventions": 0,
126+
"thrashInterventions": 0,
127+
"gateSuspensions": 0,
128+
"agentDurationMs": 22,
129+
"verifyDurationMs": 65,
130+
"verifyExitCode": 1,
131+
"overBudget": false
132+
},
133+
{
134+
"taskId": "stall-read",
135+
"title": "Stall by re-reading the same file instead of acting",
136+
"profile": "stall",
137+
"repeat": 0,
138+
"completed": false,
139+
"runStatus": "failed",
140+
"turnsUsed": 3,
141+
"toolCallCount": 3,
142+
"failedToolCalls": 0,
143+
"retryCount": 0,
144+
"compactionEvents": 0,
145+
"doomLoopInterventions": 1,
146+
"thrashInterventions": 0,
147+
"gateSuspensions": 0,
148+
"agentDurationMs": 95,
149+
"verifyDurationMs": 16,
150+
"verifyExitCode": 2,
151+
"overBudget": false,
152+
"error": "reactor error: Doom loop detected: an identical tool batch (read_file) executed 3 times consecutively"
153+
},
154+
{
155+
"taskId": "stall-read",
156+
"title": "Stall by re-reading the same file instead of acting",
157+
"profile": "stall",
158+
"repeat": 1,
159+
"completed": false,
160+
"runStatus": "failed",
161+
"turnsUsed": 3,
162+
"toolCallCount": 3,
163+
"failedToolCalls": 0,
164+
"retryCount": 0,
165+
"compactionEvents": 0,
166+
"doomLoopInterventions": 1,
167+
"thrashInterventions": 0,
168+
"gateSuspensions": 0,
169+
"agentDurationMs": 65,
170+
"verifyDurationMs": 17,
171+
"verifyExitCode": 2,
172+
"overBudget": false,
173+
"error": "reactor error: Doom loop detected: an identical tool batch (read_file) executed 3 times consecutively"
174+
}
175+
],
176+
"totals": {
177+
"tasksTotal": 4,
178+
"runsTotal": 8,
179+
"completedRuns": 4,
180+
"completionRate": 0.5,
181+
"meanTurnsToCompletion": 3,
182+
"meanAgentDurationMs": 79.25,
183+
"totalRetries": 0,
184+
"totalCompactionEvents": 0,
185+
"totalDoomLoopInterventions": 2,
186+
"totalThrashInterventions": 0
187+
}
188+
}

‎evals/completion/lib.test.ts‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
import { describe, expect, test } from "bun:test";
2+
import {
3+
CompletionReport,
4+
computeTotals,
5+
formatSummary,
6+
isCompletedRun,
7+
parseResponderScript,
8+
parseTaskSetFile,
9+
TaskResult,
10+
type CompletionReport as CompletionReportType,
11+
type TaskResult as TaskResultType,
12+
} from "./lib.js";
13+
14+
const result = (overrides: Partial<TaskResultType> = {}): TaskResultType =>
15+
TaskResult.assert({
16+
taskId: "version-endpoint",
17+
title: "Add GET /version",
18+
profile: "solve",
19+
repeat: 0,
20+
completed: true,
21+
runStatus: "completed",
22+
turnsUsed: 3,
23+
toolCallCount: 2,
24+
failedToolCalls: 0,
25+
retryCount: 0,
26+
compactionEvents: 0,
27+
doomLoopInterventions: 0,
28+
thrashInterventions: 0,
29+
gateSuspensions: 0,
30+
agentDurationMs: 1200,
31+
verifyDurationMs: 300,
32+
verifyExitCode: 0,
33+
overBudget: false,
34+
...overrides,
35+
});
36+
37+
describe("completion predicate", () => {
38+
test("completes only on clean run, green grader, and budget", () => {
39+
expect(isCompletedRun(result())).toBe(true);
40+
expect(isCompletedRun(result({ runStatus: "failed" }))).toBe(false);
41+
expect(isCompletedRun(result({ runStatus: "timeout" }))).toBe(false);
42+
expect(isCompletedRun(result({ verifyExitCode: 1 }))).toBe(false);
43+
expect(isCompletedRun(result({ overBudget: true }))).toBe(false);
44+
});
45+
});
46+
47+
describe("completion totals", () => {
48+
test("mixed outcomes yield a fractional rate and completion-only turn mean", () => {
49+
const totals = computeTotals([
50+
result({ turnsUsed: 3, agentDurationMs: 1000 }),
51+
result({
52+
taskId: "stall-read",
53+
profile: "stall",
54+
completed: false,
55+
runStatus: "failed",
56+
turnsUsed: 4,
57+
toolCallCount: 4,
58+
retryCount: 1,
59+
doomLoopInterventions: 1,
60+
verifyExitCode: 1,
61+
agentDurationMs: 3000,
62+
}),
63+
]);
64+
expect(totals.tasksTotal).toBe(2);
65+
expect(totals.runsTotal).toBe(2);
66+
expect(totals.completedRuns).toBe(1);
67+
expect(totals.completionRate).toBe(0.5);
68+
expect(totals.meanTurnsToCompletion).toBe(3);
69+
expect(totals.meanAgentDurationMs).toBe(2000);
70+
expect(totals.totalRetries).toBe(1);
71+
expect(totals.totalDoomLoopInterventions).toBe(1);
72+
});
73+
74+
test("empty results stay zero without dividing by zero", () => {
75+
const totals = computeTotals([]);
76+
expect(totals.completionRate).toBe(0);
77+
expect(totals.meanTurnsToCompletion).toBe(0);
78+
expect(totals.meanAgentDurationMs).toBe(0);
79+
});
80+
});
81+
82+
describe("boundary parsing", () => {
83+
test("rejects a task set with no tasks", () => {
84+
expect(() =>
85+
parseTaskSetFile({ version: 1, note: "x", tasks: [] }),
86+
).toThrow();
87+
});
88+
89+
test("rejects a responder script with no turns", () => {
90+
expect(() => parseResponderScript({ turns: [] })).toThrow();
91+
});
92+
93+
test("rejects a report with an out-of-range completion rate", () => {
94+
const report = {
95+
harness: "completion-baseline",
96+
version: 1,
97+
startedAt: "2026-09-14T00:00:00.000Z",
98+
finishedAt: "2026-09-14T00:01:00.000Z",
99+
commitSha: "abc",
100+
provider: "stub-scripted",
101+
model: "completion-baseline-v1",
102+
repeats: 1,
103+
taskSetVersion: 1,
104+
taskIds: ["version-endpoint"],
105+
results: [result()],
106+
totals: { ...computeTotals([result()]), completionRate: 2 },
107+
};
108+
expect(() => CompletionReport.assert(report)).toThrow();
109+
});
110+
});
111+
112+
describe("human summary", () => {
113+
test("names the harness, provenance, rate, and every run", () => {
114+
const results = [
115+
result(),
116+
result({ taskId: "stall-read", completed: false }),
117+
];
118+
const report: CompletionReportType = CompletionReport.assert({
119+
harness: "completion-baseline",
120+
version: 1,
121+
startedAt: "2026-09-14T00:00:00.000Z",
122+
finishedAt: "2026-09-14T00:01:00.000Z",
123+
commitSha: "deadbeef",
124+
provider: "stub-scripted",
125+
model: "completion-baseline-v1",
126+
repeats: 1,
127+
taskSetVersion: 1,
128+
taskIds: ["version-endpoint", "stall-read"],
129+
results,
130+
totals: computeTotals(results),
131+
});
132+
const summary = formatSummary(report);
133+
expect(summary).toContain("completion rate 50.0% (1/2 runs)");
134+
expect(summary).toContain("deadbeef");
135+
expect(summary).toContain("stub-scripted");
136+
expect(summary).toContain("version-endpoint r0: complete");
137+
expect(summary).toContain("stall-read r0: incomplete");
138+
});
139+
});

0 commit comments

Comments
 (0)