fix: bound append-heavy raw archives

docs/first-agent-history-import.md

@@ -36,6 +36,11 @@ imported; the user is asked to run a fresh discovery and approve its current
 inventory. The CLI then imports one discovered history at a time through the
 authenticated sync client. Each source fingerprint receives a deterministic
 immutable raw archive; a complete archive is hash-verified and reused on retry.
+Codex rollout archives store fixed-size content-addressed chunks, so subsequent
+snapshots of a growing transcript reuse the unchanged prefix. Background sync
+also waits five minutes for active files to settle and refreshes previously
+imported conversations no more than once every six hours. Explicit
+`ae sync once` and approved first imports are not delayed.
 The content cursor advances only after the synchronous import result is durable.
 
 The page can request cancellation at the next item boundary. Closing the page

packages/cli/README.md

@@ -78,8 +78,14 @@ into `config.yaml`, verifies the full bundle fingerprint again before reading,
 and records a resumable reconciled inventory. Changed bundles require a fresh
 preview and approval; inaccessible sources receive safe permission guidance.
 Supported history sources are Claude Code, Codex, Cowork, and local directories.
-The sync cursor and raw source archive remain local under `AE_HOME`. Raw archives
-are content-addressed and reused when an import is retried. Writes fail closed
+The sync cursor and raw source archive remain local under `AE_HOME`. Codex's
+append-heavy JSONL transcripts use fixed-size, content-addressed chunks, so a
+growing session writes only its changed tail instead of another full copy.
+Background sync waits for a file to settle and refreshes an already-imported
+conversation at most once every six hours; `ae sync once` remains immediate.
+The daemon scans every five minutes by default and reports deferred files plus
+new and reused archive bytes. Other raw archives are content-addressed and
+reused when an import is retried. Writes fail closed
 before exceeding a 256 MiB bundle limit, a 10 GiB total archive limit, or a
 10 GiB free-space reserve. Override those byte counts with
 `AE_RAW_ARCHIVE_MAX_BUNDLE_BYTES`, `AE_RAW_ARCHIVE_MAX_TOTAL_BYTES`, and

packages/cli/src/__tests__/codex-normalize.test.ts

@@ -7,6 +7,7 @@ import {
   readFileSync,
   readdirSync,
   rmSync,
+  utimesSync,
   writeFileSync,
 } from 'node:fs';
 import { tmpdir } from 'node:os';
@@ -256,21 +257,30 @@ describe('Codex conversation normalization', () => {
     expect(searchText).not.toContain('future_value');
 
     const archiveRoot = join(codexHome, '..', 'ae-home', 'raw-archive');
-    const [archiveName] = readdirSync(archiveRoot);
+    const [archiveName] = readdirSync(archiveRoot).filter((name) => !name.startsWith('.'));
     const manifest = JSON.parse(
       readFileSync(join(archiveRoot, archiveName, 'manifest.json'), 'utf8'),
     ) as {
+      version: number;
       adapter_name: string;
-      files: Array<{ path: string; archive_path: string; sha256: string }>;
+      files: Array<{
+        path: string;
+        sha256: string;
+        chunks: Array<{ archive_path: string }>;
+      }>;
     };
     const expectedHash = createHash('sha256').update(readFileSync(rolloutPath)).digest('hex');
     expect(manifest).toMatchObject({
+      version: 2,
       adapter_name: 'codex-history',
       files: [{ path: rolloutPath, sha256: expectedHash }],
     });
     expect(conversation.source_sha256).toBe(expectedHash);
     expect(result.sourceFingerprint).toBe(expectedHash);
-    expect(readFileSync(join(archiveRoot, archiveName, manifest.files[0].archive_path)))
+    const archivedBytes = Buffer.concat(
+      manifest.files[0].chunks.map((chunk) => readFileSync(join(archiveRoot, chunk.archive_path))),
+    );
+    expect(archivedBytes)
       .toEqual(readFileSync(rolloutPath));
   });
 
@@ -437,6 +447,46 @@ describe('Codex conversation normalization', () => {
       source_identifier: 'openai_codex:codex:codex-session-915',
       'metadata.sync.adapter_name': 'codex-history',
     });
-    expect(readdirSync(join(codexHome, '..', 'ae-home', 'raw-archive'))).toHaveLength(1);
+    const archiveRoot = join(codexHome, '..', 'ae-home', 'raw-archive');
+    expect(readdirSync(archiveRoot).filter((name) => !name.startsWith('.'))).toHaveLength(1);
+    expect(readdirSync(archiveRoot)).toContain('.chunks');
+  });
+
+  it('throttles changed conversation snapshots only in the background daemon', async () => {
+    const { codexHome, rolloutPath, database } = makeCodexFixture();
+    const cursorFile = join(codexHome, 'cursor.json');
+    const client = {
+      submitSyncImport: vi.fn((request: ImportRequest) => Promise.resolve({
+        success: true as const,
+        data: {
+          totalItems: request.items.length,
+          completedItems: request.items.length,
+          failedItems: 0,
+          contentIds: ['content-codex'],
+          failures: [],
+        },
+      })),
+    };
+    const now = Date.now();
+    const stableTime = new Date(now - 10 * 60 * 1000);
+    utimesSync(rolloutPath, stableTime, stableTime);
+    const options = {
+      sourceId: 'codex' as const,
+      paths: [rolloutPath], cursorFile, batchSize: 1, client,
+    };
+
+    const first = await runSyncOnce({ ...options, enforceArchiveThrottle: true, nowMs: now });
+    writeFileSync(rolloutPath, `${readFileSync(rolloutPath, 'utf8')}\n`);
+    utimesSync(rolloutPath, stableTime, stableTime);
+    const deferred = await runSyncOnce({
+      ...options, enforceArchiveThrottle: true, nowMs: now + 20 * 60 * 1000,
+    });
+    const manual = await runSyncOnce(options);
+    closeTestDatabase(database);
+
+    expect(first.turnsImported).toBe(1);
+    expect(deferred).toMatchObject({ turnsImported: 0, deferredFiles: 1 });
+    expect(manual.turnsImported).toBe(1);
+    expect(client.submitSyncImport).toHaveBeenCalledTimes(2);
   });
 });

packages/cli/src/__tests__/raw-archive.test.ts

@@ -1,14 +1,16 @@
 import { createHash } from 'node:crypto';
-import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { appendFileSync, existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
 import { tmpdir } from 'node:os';
-import { join } from 'node:path';
+import { dirname, join } from 'node:path';
 import { afterEach, describe, expect, it } from 'vitest';
 import {
   RawArchiveCapacityError,
   RawArchiveManifestSchema,
   applyRawArchiveRetention,
   inspectRawArchive,
   planRawArchiveRetention,
+  readRawArchiveFile,
+  writeChunkedRawArchive,
   writeRawArchive,
 } from '../sync/raw-archive.js';
 
@@ -30,6 +32,48 @@ afterEach(() => {
 });
 
 describe('writeRawArchive', () => {
+  it('deduplicates unchanged chunks when an append-only transcript grows', async () => {
+    const root = makeTempDir();
+    process.env.AE_HOME = join(root, 'ae-home');
+    const sourcePath = join(root, 'large.jsonl');
+    const testChunkBytes = 64 * 1024;
+    const initial = Buffer.alloc(testChunkBytes + 1024, 97);
+    writeFileSync(sourcePath, initial);
+
+    const first = await writeChunkedRawArchive(sourcePath, {
+      adapterName: 'append-only-test', adapterVersion: '1.0.0',
+      chunkBytes: testChunkBytes,
+    });
+    appendFileSync(sourcePath, Buffer.alloc(128, 98));
+    const second = await writeChunkedRawArchive(sourcePath, {
+      adapterName: 'append-only-test', adapterVersion: '1.0.0',
+      chunkBytes: testChunkBytes,
+    });
+
+    expect(first.manifest.version).toBe(2);
+    expect(first.newlyArchivedBytes).toBe(initial.length);
+    expect(second.newlyArchivedBytes).toBeLessThanOrEqual(testChunkBytes + 128);
+    expect(second.reusedBytes).toBeGreaterThanOrEqual(testChunkBytes);
+    expect(await readRawArchiveFile(second, sourcePath)).toEqual(readFileSync(sourcePath));
+    expect(readdirSync(second.archiveDir)).toEqual(['manifest.json']);
+  });
+
+  it('rejects a tampered shared chunk before returning archived bytes', async () => {
+    const root = makeTempDir();
+    process.env.AE_HOME = join(root, 'ae-home');
+    const sourcePath = join(root, 'source.jsonl');
+    writeFileSync(sourcePath, '{"message":"integrity"}\n');
+    const archive = await writeChunkedRawArchive(sourcePath, {
+      adapterName: 'append-only-test', adapterVersion: '1.0.0',
+    });
+    const entry = archive.manifest.files[0];
+    const chunk = entry?.chunks[0];
+    expect(chunk).toBeDefined();
+    writeFileSync(join(dirname(archive.archiveDir), chunk!.archive_path), 'tampered');
+
+    await expect(readRawArchiveFile(archive, sourcePath)).rejects.toThrow(/integrity/i);
+  });
+
   it('requires adapter versioning in the manifest contract', () => {
     expect(RawArchiveManifestSchema.safeParse({
       version: 1,

packages/cli/src/commands/sync.ts

@@ -54,7 +54,7 @@ import {
 } from '../sync/raw-archive.js';
 
 const DEFAULT_SYNC_BATCH_SIZE = 25;
-const DEFAULT_POLL_INTERVAL_SECONDS = 15;
+const DEFAULT_POLL_INTERVAL_SECONDS = 300;
 
 interface SyncCommandOptions {
   source?: string;
@@ -137,7 +137,9 @@ function printLoopSummary(summary: SyncRunSummary): void {
   console.log(
     `[${timestamp}] source=${summary.sourceId} files=${summary.filesScanned} ` +
     `found=${summary.turnsFound} imported=${summary.turnsImported} ` +
-    `failed=${summary.failedItems} parseErrors=${summary.parseErrors}`
+    `deferred=${summary.deferredFiles} failed=${summary.failedItems} ` +
+    `parseErrors=${summary.parseErrors} archiveWritten=${summary.archiveBytesWritten} ` +
+    `archiveReused=${summary.archiveBytesReused}`
   );
 }