feat: a script re-prepares a pull request with another model and compares the findings with the bundle it has

development.md

@@ -126,6 +126,44 @@ npm run test:watch -w @diffity/parser
 npm run test:watch -w @diffity/ui
 ```
 
+## Comparing review models
+
+`scripts/inbox-compare.ts` re-prepares a pull request the inbox has already reviewed, with a
+different model or effort, and puts the two sets of findings side by side. It is how a change to
+`agent.model` is decided: by what the candidate finds, not by what it costs.
+
+```bash
+npm run build   # the script runs the built CLI, so build first
+npx tsx scripts/inbox-compare.ts NaturalCycles/NCBackend3#14550 --model opus
+npx tsx scripts/inbox-compare.ts NaturalCycles/NCBackend3#14550 --effort medium --out /tmp/14550.md
+```
+
+The baseline is the newest bundle for that pull request under `~/.diffity/inbox/bundles`
+(`--bundles-dir` to look elsewhere, `--head <sha>` to pick an older one). The candidate gets a
+scratch worktree and its own diffity data directory under a fresh temp directory (`--scratch` to
+name it), so nothing is written into `~/.diffity` and a running `diffity inbox` is undisturbed. The
+worktree is removed afterwards unless `--keep` is passed, which leaves it in place so the
+candidate's own review can be opened in the browser. One invocation is one agent run, and it takes
+as long as a real preparation — up to half an hour.
+
+The candidate is pinned to the baseline's head, so a pull request that has moved on since — or has
+merged — can still be compared. An earlier head usually comes along with the pull request's own
+ref; when it does not, it is fetched by sha, which the forge serves for any commit reachable from a
+ref it advertises. A head that was force-pushed away is gone for good, and the run is refused with
+`the baseline's head <sha> is no longer reachable from origin`.
+
+Reading the table: each severity row is `baseline count | reproduced, new`. *Reproduced* means a
+candidate finding landed on the same file with an overlapping line range — a one-line finding
+counts as its line give or take two. *New* counts candidate findings no baseline finding covers;
+some are real, some are noise, which is what the finding list underneath is for. The `cost / time`
+row is the candidate's own run: two models are not comparable on the baseline's, which predates
+the run log. Exit code 0 is a completed comparison, 2 a candidate that skipped or failed, 1 a
+usage error. `--json` prints the same numbers as one object.
+
+The baseline bundles were prepared by the old pipeline, which loaded the reviewer's own settings,
+skills and MCP servers and let the agent run the repository's toolchain. A difference between the
+columns is therefore prompt *and* model, not model alone.
+
 ## CLI Usage (for reference while developing)
 
 ```bash

package.json

@@ -19,7 +19,7 @@
     "link-dev": "tsx scripts/link-dev.ts",
     "dev": "tsx scripts/dev.ts",
     "test:scripts": "vitest run scripts",
-    "typecheck": "npm run typecheck -w @diffity/parser && npm run typecheck -w @diffity/api && npm run typecheck -w @diffity/git && npm run typecheck -w @diffity/github && npm run typecheck -w @diffity/ui && npm run typecheck -w @naturalcycles/diffity"
+    "typecheck": "npm run typecheck -w @diffity/parser && npm run typecheck -w @diffity/api && npm run typecheck -w @diffity/git && npm run typecheck -w @diffity/github && npm run typecheck -w @diffity/ui && npm run typecheck -w @naturalcycles/diffity && tsc -p tsconfig.scripts.json"
   },
   "keywords": [
     "git",

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@diffity/api",
-  "version": "0.10.25",
+  "version": "0.10.26",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@naturalcycles/diffity",
-  "version": "0.10.25",
+  "version": "0.10.26",
   "description": "Agent-agnostic, GitHub-style diff viewer and code review tool with a live agent loop",
   "type": "module",
   "bin": {

packages/cli/src/inbox/prepare.ts

@@ -94,6 +94,11 @@ export function logsDir(): string {
 export interface PrepareOpts {
   /** The reviewer asked for this one by name, so the filter does not get a say. */
   bumped?: boolean;
+  /**
+   * Review this commit rather than wherever the pull request has got to. Only a comparison against
+   * an earlier review sets it; the daemon always takes the current head.
+   */
+  pinHead?: string;
 }
 
 export async function preparePr(snapshot: PrSnapshot, config: InboxConfig, deps: PrepareDeps, opts: PrepareOpts = {}): Promise<PrepareResult> {
@@ -107,7 +112,7 @@ export async function preparePr(snapshot: PrSnapshot, config: InboxConfig, deps:
   let head: string;
   let diffRef: string;
   try {
-    ({ head, diffRef } = await prepareWorktree(clone, dest, snapshot, snapshot.baseRef));
+    ({ head, diffRef } = await prepareWorktree(clone, dest, snapshot, snapshot.baseRef, opts.pinHead));
   } catch (err) {
     return { kind: 'failed', failure: 'worktree', reason: err instanceof Error ? err.message : String(err), worktree: null, logPath: null, run };
   }

packages/cli/src/inbox/worktree.ts

@@ -38,8 +38,11 @@ async function runGit(cwd: string, args: string[]): Promise<string> {
  * diff against — the fetched base, so a diffity session over the worktree shows the same change as
  * the pull request without asking the forge anything. Idempotent and self-healing: an existing
  * worktree, even one a killed agent left dirty, is forced to the new head rather than re-created.
+ *
+ * `pinHead` cuts the worktree at that commit instead of wherever the pull request has got to,
+ * which is how a review is re-run against the head an earlier one was written at.
  */
-export async function prepareWorktree(clone: string, dest: string, ref: PrRef, baseRef: string): Promise<{ head: string; diffRef: string }> {
+export async function prepareWorktree(clone: string, dest: string, ref: PrRef, baseRef: string, pinHead?: string): Promise<{ head: string; diffRef: string }> {
   if (!existsSync(clone)) {
     throw new Error(`No local clone at ${clone}. Clone ${ref.owner}/${ref.repo} there first.`);
   }
@@ -49,7 +52,9 @@ export async function prepareWorktree(clone: string, dest: string, ref: PrRef, b
   await requireMatchingOrigin(clone, ref);
 
   await runGit(clone, ['fetch', 'origin', `refs/pull/${ref.number}/head`]);
-  const head = await runGit(clone, ['rev-parse', 'FETCH_HEAD']);
+  const head = pinHead === undefined
+    ? await runGit(clone, ['rev-parse', 'FETCH_HEAD'])
+    : await reachable(clone, pinHead);
   // `refs/heads/` so a tag sharing the branch's name cannot be fetched in its place.
   await runGit(clone, ['fetch', 'origin', `refs/heads/${baseRef}`]);
   const diffRef = await runGit(clone, ['rev-parse', 'FETCH_HEAD']);
@@ -77,6 +82,25 @@ export async function prepareWorktree(clone: string, dest: string, ref: PrRef, b
   return { head, diffRef };
 }
 
+/**
+ * The pinned commit, as a full sha, with the object in the clone. Fetching the pull request's ref
+ * usually brings it along already — an earlier head is an ancestor of a later one unless the branch
+ * was rewritten — and otherwise the forge serves any commit reachable from a ref it advertises, so
+ * it is asked for by sha. A force-push is what puts a commit out of reach for good.
+ */
+async function reachable(clone: string, pinHead: string): Promise<string> {
+  try {
+    await runGit(clone, ['cat-file', '-e', `${pinHead}^{commit}`]);
+  } catch {
+    try {
+      await runGit(clone, ['fetch', 'origin', pinHead]);
+    } catch {
+      throw new Error(`the baseline's head ${pinHead.slice(0, 12)} is no longer reachable from origin (force-pushed?)`);
+    }
+  }
+  return runGit(clone, ['rev-parse', pinHead]);
+}
+
 /** The clone must actually be the pull request's repository, not another of the same name. */
 async function requireMatchingOrigin(clone: string, ref: PrRef): Promise<void> {
   let url: string;

packages/cli/tests/inbox-prepare.test.ts

@@ -483,6 +483,25 @@ describe('the inbox JSON server', () => {
     store.close();
   });
 
+  it('reviews a pinned head the pull request has moved past', async () => {
+    // A second push, so the snapshot's head is no longer where the pull request points.
+    const upstream = join(root, 'remotes', 'o', 'demo');
+    writeFileSync(join(upstream, 'b.ts'), 'const b = 2;\n');
+    git(upstream, ['add', '.']);
+    git(upstream, ['commit', '-m', 'more']);
+    git(upstream, ['update-ref', 'refs/pull/4/head', 'HEAD']);
+    const moved = git(upstream, ['rev-parse', 'HEAD']);
+
+    const result = await preparePr(snapshot(), config(), deps(), { pinHead: head });
+
+    expect(result.kind).toBe('prepared');
+    if (result.kind !== 'prepared') return;
+    expect(result.headSha).toBe(head);
+    expect(result.headSha).not.toBe(moved);
+    expect(existsSync(join(result.worktree, 'b.ts'))).toBe(false);
+    expect(result.bundlePath).toContain(head.slice(0, 12));
+  });
+
   it('sets the filter aside for a bumped pull request', async () => {
     prompts = [];
     const withFilter = { ...config(), filter: 'Skip payments-focused PRs' };

packages/cli/tests/inbox-worktree.test.ts

@@ -6,6 +6,7 @@ import { tmpdir } from 'node:os';
 import { prepareWorktree, removeWorktree } from '../src/inbox/worktree.js';
 
 let root: string;
+let upstream: string;
 let clone: string;
 let dest: string;
 let head: string;
@@ -17,7 +18,7 @@ function git(cwd: string, args: string[]): string {
 
 beforeEach(() => {
   root = mkdtempSync(join(tmpdir(), 'diffity-worktree-'));
-  const upstream = join(root, 'remotes', 'o', 'demo');
+  upstream = join(root, 'remotes', 'o', 'demo');
   execFileSync('git', ['init', '-b', 'main', upstream], { stdio: 'pipe' });
   git(upstream, ['config', 'user.email', 't@t']);
   git(upstream, ['config', 'user.name', 'T']);
@@ -80,6 +81,63 @@ describe('prepareWorktree', () => {
     expect(existsSync(join(dest, 'debris.txt'))).toBe(false);
     expect(git(dest, ['rev-parse', 'HEAD'])).toBe(head);
   });
+
+  describe('with a pinned head', () => {
+    /** A second push on the pull request, leaving the head captured in `head` behind. */
+    function pushOnTop(): string {
+      writeFileSync(join(upstream, 'b.ts'), 'const b = 2;\n');
+      git(upstream, ['add', '.']);
+      git(upstream, ['commit', '-m', 'more']);
+      git(upstream, ['update-ref', `refs/pull/${ref.number}/head`, 'HEAD']);
+      return git(upstream, ['rev-parse', 'HEAD']);
+    }
+
+    it('cuts the worktree at an earlier head the pull request has moved past', async () => {
+      const moved = pushOnTop();
+
+      const cut = await prepareWorktree(clone, dest, ref, 'main', head);
+
+      expect(cut.head).toBe(head);
+      expect(git(dest, ['rev-parse', 'HEAD'])).toBe(head);
+      // The later commit's file is not in the tree, so the review is about the pinned code.
+      expect(existsSync(join(dest, 'b.ts'))).toBe(false);
+      expect(moved).not.toBe(head);
+    });
+
+    it('takes the pull request as it stands when nothing is pinned', async () => {
+      const moved = pushOnTop();
+
+      const cut = await prepareWorktree(clone, dest, ref, 'main');
+
+      expect(cut.head).toBe(moved);
+      expect(existsSync(join(dest, 'b.ts'))).toBe(true);
+    });
+
+    it('asks origin by sha for a pinned commit the clone has never fetched', async () => {
+      // What GitHub allows: any commit reachable from a ref it advertises can be asked for by sha.
+      git(upstream, ['config', 'uploadpack.allowReachableSHA1InWant', 'true']);
+      // On a branch of its own, made after the clone, so no fetch of main or the pull ref brings it.
+      git(upstream, ['checkout', '-q', '-b', 'other']);
+      writeFileSync(join(upstream, 'c.ts'), 'const c = 3;\n');
+      git(upstream, ['add', '.']);
+      git(upstream, ['commit', '-m', 'elsewhere']);
+      const elsewhere = git(upstream, ['rev-parse', 'HEAD']);
+      expect(() => git(clone, ['cat-file', '-e', `${elsewhere}^{commit}`])).toThrow();
+
+      const cut = await prepareWorktree(clone, dest, ref, 'main', elsewhere);
+
+      expect(cut.head).toBe(elsewhere);
+      expect(existsSync(join(dest, 'c.ts'))).toBe(true);
+    });
+
+    it('says so when the pinned head is gone from origin', async () => {
+      const forcePushedAway = 'deadbeef'.repeat(5);
+
+      await expect(prepareWorktree(clone, dest, ref, 'main', forcePushedAway))
+        .rejects.toThrow(/the baseline's head deadbeefdead is no longer reachable from origin \(force-pushed\?\)/);
+      expect(existsSync(dest)).toBe(false);
+    });
+  });
 });
 
 describe('removeWorktree', () => {

packages/git/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@diffity/git",
-  "version": "0.10.25",
+  "version": "0.10.26",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/github/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@diffity/github",
-  "version": "0.10.25",
+  "version": "0.10.26",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/parser/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@diffity/parser",
-  "version": "0.10.25",
+  "version": "0.10.26",
   "private": true,
   "type": "module",
   "main": "./dist/index.js",

packages/ui/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@diffity/ui",
-  "version": "0.10.25",
+  "version": "0.10.26",
   "type": "module",
   "private": true,
   "scripts": {