From 2597081e645eb6447a90b935f21573e996e7a8a5 Mon Sep 17 00:00:00 2001 From: Jay/Fienna Liang Date: Wed, 2 Sep 2026 19:27:29 +0800 Subject: [PATCH] feat(checkpoint): add portable directory policy --- docs/architecture/core-layering.md | 9 +- docs/architecture/durable-state.md | 10 + docs/project-posture.md | 4 + .../unit/core/memory-checkpoint.test.ts | 42 ++ .../portable-directory-checkpoint.test.ts | 225 +++++++++++ src/advanced.ts | 17 + src/core/checkpoint/index.ts | 1 + .../checkpoint/portable-directory/README.md | 83 ++++ .../checkpoint/portable-directory/codec.ts | 74 ++++ .../checkpoint/portable-directory/errors.ts | 43 ++ .../checkpoint/portable-directory/index.ts | 17 + .../checkpoint/portable-directory/policy.ts | 51 +++ .../checkpoint/portable-directory/schemas.ts | 11 + .../checkpoint/portable-directory/service.ts | 377 ++++++++++++++++++ .../checkpoint/portable-directory/types.ts | 12 + src/core/memory/README.md | 7 +- src/core/memory/checkpoint/README.md | 11 + src/core/memory/checkpoint/codec.ts | 66 +-- .../memory/checkpoint/directory-policy.ts | 22 + src/core/memory/checkpoint/schemas.ts | 10 +- src/core/memory/checkpoint/service.ts | 116 ++---- 21 files changed, 1059 insertions(+), 149 deletions(-) create mode 100644 src/__tests__/unit/core/portable-directory-checkpoint.test.ts create mode 100644 src/core/checkpoint/index.ts create mode 100644 src/core/checkpoint/portable-directory/README.md create mode 100644 src/core/checkpoint/portable-directory/codec.ts create mode 100644 src/core/checkpoint/portable-directory/errors.ts create mode 100644 src/core/checkpoint/portable-directory/index.ts create mode 100644 src/core/checkpoint/portable-directory/policy.ts create mode 100644 src/core/checkpoint/portable-directory/schemas.ts create mode 100644 src/core/checkpoint/portable-directory/service.ts create mode 100644 src/core/checkpoint/portable-directory/types.ts create mode 100644 src/core/memory/checkpoint/directory-policy.ts diff --git a/docs/architecture/core-layering.md b/docs/architecture/core-layering.md index 65e5890a..d8804fa8 100644 --- a/docs/architecture/core-layering.md +++ b/docs/architecture/core-layering.md @@ -31,7 +31,7 @@ src/core/agent Layer 1: Infrastructure and domain primitives src/core/llm, src/core/tools, src/core/trace, src/core/auth, src/core/approvals, -src/core/commands +src/core/commands, src/core/checkpoint Layer 0: Shared types and utilities src/core/types, src/core/utils, src/core/config @@ -84,6 +84,13 @@ service behavior. If a utility clearly belongs to one domain, place it in that domain instead; for example, step-budget behavior lives under `src/core/agent` rather than generic `src/core/utils`. +`src/core/checkpoint/portable-directory` is the lower shared filesystem +primitive for domain-owned checkpoints. It owns safe relative paths, selected +regular-file integrity, explicit resource policies, and staged local restore. +Memory and future directory-backed domains retain separate scope, manifest, +store, and lifecycle contracts above it; do not turn this primitive into a +universal persistence provider. + ## Current Violations And Improvement Areas These are known cleanup directions, not blockers for every feature: diff --git a/docs/architecture/durable-state.md b/docs/architecture/durable-state.md index b41109ea..4306e34f 100644 --- a/docs/architecture/durable-state.md +++ b/docs/architecture/durable-state.md @@ -329,6 +329,16 @@ boundaries. A shutdown checkpoint may reduce loss, but must not be the sole commit path. An official object-store adapter, periodic long-session trigger, and Execution Host recovery integration are not implemented yet. +Memory's local capture and restore delegate to the reusable +[`PortableDirectoryCheckpointService`](../../src/core/checkpoint/portable-directory/README.md). +That lower primitive requires an explicit selection and resource policy, +rejects unsafe paths and symlinks, validates per-file integrity, and stages a +complete directory before rename. It does not define durable scope, generation +identity, manifest compare-and-swap, or execution settlement. Memory v1 uses +compatibility ceilings so this extraction does not silently reject previously +accepted memory checkpoints; a future working-directory specialization must +choose narrower operational limits in its own public contract. + ### Approval policy and project configuration Remembered project approvals live in `command-approvals.json` through diff --git a/docs/project-posture.md b/docs/project-posture.md index c94fa169..4fd7e59e 100644 --- a/docs/project-posture.md +++ b/docs/project-posture.md @@ -106,6 +106,10 @@ See `docs/guides/programmatic/component-model.md` for the user-facing model. - `src/core/runtime/` owns host-facing runtime boundaries, default tool assembly, credentials, workspace catalogs, daemon discovery, and evented single-run execution over `src/core/agent/`. +- `src/core/checkpoint/` owns lower shared checkpoint primitives such as safe, + bounded portable-directory capture and staged restore. Domain scopes, + manifests, stores, and lifecycle timing remain in memory, heartbeat, or the + future domain that gives those files meaning. - `src/core/heartbeat/` owns autonomous runner cycles, heartbeat scheduling, checkpoint reuse, and heartbeat task/run views. - `src/core/chat/engine/` owns persisted conversation sessions, turns, diff --git a/src/__tests__/unit/core/memory-checkpoint.test.ts b/src/__tests__/unit/core/memory-checkpoint.test.ts index 63a9c8a5..b38e6d28 100644 --- a/src/__tests__/unit/core/memory-checkpoint.test.ts +++ b/src/__tests__/unit/core/memory-checkpoint.test.ts @@ -3,6 +3,7 @@ import { join } from 'node:path'; import { tmpdir } from 'node:os'; import { afterEach, describe, expect, it } from 'vitest'; import { + MemoryCheckpointCodec, MemoryCheckpointConflictError, MemoryCheckpointCorruptionError, MemoryCheckpointGenerationIdSchema, @@ -100,6 +101,47 @@ describe('MemoryCheckpointService', () => { await expect(stat(join(fixture.memoryRoot, 'credentials.json'))).rejects.toMatchObject({ code: 'ENOENT' }); }); + it('preserves the exact released memory-v1 generation and manifest bytes', () => { + const generation = MemoryCheckpointCodec.createGeneration({ + scopeId, + generationId: MemoryCheckpointGenerationIdSchema.parse('generation-1'), + createdAt: '2026-08-26T00:00:00.000Z', + files: [MemoryCheckpointCodec.createFile('README.md', Buffer.from('# Memory\n'))], + }); + const manifest = MemoryCheckpointCodec.createManifest({ + generation, + committedAt: '2026-08-26T00:00:00.000Z', + }); + + expect(MemoryCheckpointCodec.serializeGeneration(generation)).toBe(`{ + "kind": "heddle-memory-checkpoint-generation", + "schemaVersion": 1, + "scopeId": "memory-v1-f6c317c8362c7f6a261782e0c296239075d478fdbf9e861bc6a7a5c3ffc50a97", + "generationId": "generation-1", + "createdAt": "2026-08-26T00:00:00.000Z", + "files": [ + { + "path": "README.md", + "contentBase64": "IyBNZW1vcnkK", + "byteLength": 9, + "sha256": "d7870cdadd1ac3b46461cce0776275aeb54f15f19338e597fafd0f277b1f0070" + } + ] +} +`); + expect(MemoryCheckpointCodec.serializeManifest(manifest)).toBe(`{ + "kind": "heddle-memory-checkpoint-manifest", + "schemaVersion": 1, + "scopeId": "memory-v1-f6c317c8362c7f6a261782e0c296239075d478fdbf9e861bc6a7a5c3ffc50a97", + "generationId": "generation-1", + "generationSha256": "e114fc81376990386a2bac155bd20dd7875c4595b056f7f3fa6a0790ef0d0a3b", + "fileCount": 1, + "totalBytes": 9, + "committedAt": "2026-08-26T00:00:00.000Z" +} +`); + }); + it('rejects corrupt durable content before creating a local working copy', async () => { const fixture = await createFixture(); const store = new InMemoryCheckpointStore(); diff --git a/src/__tests__/unit/core/portable-directory-checkpoint.test.ts b/src/__tests__/unit/core/portable-directory-checkpoint.test.ts new file mode 100644 index 00000000..3138ebee --- /dev/null +++ b/src/__tests__/unit/core/portable-directory-checkpoint.test.ts @@ -0,0 +1,225 @@ +import { + lstat, + mkdir, + mkdtemp, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { + PortableDirectoryCheckpointCodec, + PortableDirectoryCheckpointCorruptionError, + PortableDirectoryCheckpointPolicy, + PortableDirectoryCheckpointPolicyError, + PortableDirectoryCheckpointRestoreTargetError, + PortableDirectoryCheckpointService, + type PortableDirectoryCheckpointFile, + type PortableDirectoryCheckpointLimits, +} from '@/core/checkpoint/portable-directory/index.js'; + +const GENEROUS_LIMITS: PortableDirectoryCheckpointLimits = { + maxFileCount: 20, + maxFileBytes: 1024, + maxTotalBytes: 4096, +}; + +describe('PortableDirectoryCheckpointService', () => { + const temporaryRoots: string[] = []; + + afterEach(async () => { + await Promise.all(temporaryRoots.splice(0).map(path => rm(path, { recursive: true, force: true }))); + }); + + it('round-trips selected binary files in canonical path order', async () => { + const directoryRoot = await createDirectoryRoot(); + await mkdir(join(directoryRoot, 'a'), { recursive: true }); + await mkdir(join(directoryRoot, 'excluded'), { recursive: true }); + await writeFile(join(directoryRoot, 'a.md'), Buffer.from('top-level')); + await writeFile(join(directoryRoot, 'a', 'binary.dat'), Buffer.from([0x00, 0xff, 0x10])); + await writeFile(join(directoryRoot, 'z.txt'), Buffer.from('last')); + await writeFile(join(directoryRoot, 'excluded', 'secret.txt'), Buffer.from('not portable')); + const service = createService(directoryRoot, { + includeFile: path => !path.startsWith('excluded/'), + }); + + const files = await service.capture(); + + expect(files.map(file => file.path)).toEqual(['a.md', 'a/binary.dat', 'z.txt']); + await rm(directoryRoot, { recursive: true }); + await expect(service.restore(files)).resolves.toBe(directoryRoot); + await expect(readFile(join(directoryRoot, 'a', 'binary.dat'))) + .resolves.toEqual(Buffer.from([0x00, 0xff, 0x10])); + await expect(lstat(join(directoryRoot, 'excluded'))).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('recognizes canonical relative paths across POSIX and Windows syntax', () => { + expect(PortableDirectoryCheckpointCodec.isSafeRelativePath('notes/current.md')).toBe(true); + expect([ + '', + '.', + '../escape.txt', + 'notes/../escape.txt', + '/absolute.txt', + 'C:/absolute.txt', + 'C:drive-relative.txt', + 'notes\\windows.txt', + 'notes//duplicate-separator.txt', + ].map(path => PortableDirectoryCheckpointCodec.isSafeRelativePath(path))) + .toEqual([false, false, false, false, false, false, false, false, false]); + }); + + it.runIf(process.platform !== 'win32')('rejects unsafe relative names and symbolic links during capture', async () => { + const unsafeRoot = await createDirectoryRoot(); + await writeFile(join(unsafeRoot, 'unsafe\\name.txt'), 'unsafe'); + await expect(createService(unsafeRoot).capture()).rejects.toMatchObject({ + code: 'PORTABLE_DIRECTORY_CHECKPOINT_CAPTURE_ERROR', + detail: expect.stringContaining('unsafe relative file path'), + }); + + const symlinkRoot = await createDirectoryRoot(); + await writeFile(join(symlinkRoot, 'source.txt'), 'source'); + await symlink(join(symlinkRoot, 'source.txt'), join(symlinkRoot, 'link.txt')); + await expect(createService(symlinkRoot).capture()).rejects.toMatchObject({ + code: 'PORTABLE_DIRECTORY_CHECKPOINT_CAPTURE_ERROR', + detail: expect.stringContaining('symbolic links'), + }); + }); + + it.each([ + { + name: 'file count', + limits: { maxFileCount: 1, maxFileBytes: 10, maxTotalBytes: 20 }, + files: [['one.txt', '1'], ['two.txt', '2']], + detail: 'file count 2 exceeds limit 1', + }, + { + name: 'per-file bytes', + limits: { maxFileCount: 2, maxFileBytes: 2, maxTotalBytes: 20 }, + files: [['large.txt', '123']], + detail: 'file large.txt has 3 bytes, exceeding limit 2', + }, + { + name: 'total bytes', + limits: { maxFileCount: 2, maxFileBytes: 4, maxTotalBytes: 5 }, + files: [['one.txt', '123'], ['two.txt', '456']], + detail: 'total bytes 6 exceed limit 5', + }, + ])('fails closed when capture exceeds the $name limit', async ({ limits, files, detail }) => { + const directoryRoot = await createDirectoryRoot(); + await Promise.all(files.map(([path, content]) => writeFile(join(directoryRoot, path), content))); + + await expect(createService(directoryRoot, { limits }).capture()).rejects.toMatchObject({ + code: 'PORTABLE_DIRECTORY_CHECKPOINT_CAPTURE_ERROR', + detail, + }); + }); + + it.each([ + { + name: 'file count', + limits: { maxFileCount: 1, maxFileBytes: 10, maxTotalBytes: 20 }, + files: [createFile('one.txt', '1'), createFile('two.txt', '2')], + detail: 'file count 2 exceeds limit 1', + }, + { + name: 'per-file bytes', + limits: { maxFileCount: 2, maxFileBytes: 2, maxTotalBytes: 20 }, + files: [createFile('large.txt', '123')], + detail: 'file large.txt has 3 bytes, exceeding limit 2', + }, + { + name: 'total bytes', + limits: { maxFileCount: 2, maxFileBytes: 4, maxTotalBytes: 5 }, + files: [createFile('one.txt', '123'), createFile('two.txt', '456')], + detail: 'total bytes 6 exceed limit 5', + }, + ])('fails closed when restore exceeds the $name limit', async ({ limits, files, detail }) => { + const directoryRoot = await createAbsentDirectoryRoot(); + + await expect(createService(directoryRoot, { limits }).restore(files)).rejects.toMatchObject({ + code: 'PORTABLE_DIRECTORY_CHECKPOINT_CORRUPTION', + detail, + }); + await expect(lstat(directoryRoot)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('validates integrity, policy, and path conflicts before creating the target', async () => { + const corruptRoot = await createAbsentDirectoryRoot(); + const valid = createFile('valid.txt', 'valid'); + await expect(createService(corruptRoot).restore([ + { ...valid, contentBase64: Buffer.from('corrupt').toString('base64') }, + ])).rejects.toBeInstanceOf(PortableDirectoryCheckpointCorruptionError); + await expect(lstat(corruptRoot)).rejects.toMatchObject({ code: 'ENOENT' }); + + const unsafeRoot = await createAbsentDirectoryRoot(); + await expect(createService(unsafeRoot).restore([{ ...valid, path: '../escape.txt' }])) + .rejects.toBeInstanceOf(PortableDirectoryCheckpointCorruptionError); + await expect(lstat(unsafeRoot)).rejects.toMatchObject({ code: 'ENOENT' }); + + const conflictRoot = await createAbsentDirectoryRoot(); + await expect(createService(conflictRoot).restore([ + createFile('entry', 'file'), + createFile('entry/child.txt', 'child'), + ])).rejects.toMatchObject({ + code: 'PORTABLE_DIRECTORY_CHECKPOINT_CORRUPTION', + detail: 'file path conflicts with descendant: entry', + }); + await expect(lstat(conflictRoot)).rejects.toMatchObject({ code: 'ENOENT' }); + }); + + it('restores through an empty target but does not merge with a populated target', async () => { + const emptyRoot = await createDirectoryRoot(); + await expect(createService(emptyRoot).restore([createFile('restored.txt', 'restored')])) + .resolves.toBe(emptyRoot); + await expect(readFile(join(emptyRoot, 'restored.txt'), 'utf8')).resolves.toBe('restored'); + + const populatedRoot = await createDirectoryRoot(); + await writeFile(join(populatedRoot, 'existing.txt'), 'keep'); + + await expect(createService(populatedRoot).restore([createFile('next.txt', 'next')])) + .rejects.toBeInstanceOf(PortableDirectoryCheckpointRestoreTargetError); + await expect(readFile(join(populatedRoot, 'existing.txt'), 'utf8')).resolves.toBe('keep'); + }); + + it('requires every policy limit to be an explicit non-negative safe integer', () => { + expect(() => new PortableDirectoryCheckpointPolicy({ + includeFile: () => true, + limits: { maxFileCount: Number.POSITIVE_INFINITY, maxFileBytes: 1, maxTotalBytes: 1 }, + })).toThrow(PortableDirectoryCheckpointPolicyError); + }); + + async function createDirectoryRoot(): Promise { + const root = await mkdtemp(join(tmpdir(), 'heddle-portable-directory-')); + temporaryRoots.push(root); + return root; + } + + async function createAbsentDirectoryRoot(): Promise { + const parent = await createDirectoryRoot(); + return join(parent, 'restored'); + } +}); + +function createService( + directoryRoot: string, + options: { + limits?: PortableDirectoryCheckpointLimits; + includeFile?: (path: string) => boolean; + } = {}, +): PortableDirectoryCheckpointService { + return new PortableDirectoryCheckpointService( + directoryRoot, + new PortableDirectoryCheckpointPolicy({ + limits: options.limits ?? GENEROUS_LIMITS, + includeFile: options.includeFile ?? (() => true), + }), + ); +} + +function createFile(path: string, content: string): PortableDirectoryCheckpointFile { + return PortableDirectoryCheckpointCodec.createFile(path, Buffer.from(content)); +} diff --git a/src/advanced.ts b/src/advanced.ts index e09b6b05..ba85628b 100644 --- a/src/advanced.ts +++ b/src/advanced.ts @@ -151,6 +151,23 @@ export { } from './core/observability/index.js'; export { buildSystemPrompt } from './core/prompts/system-prompt.js'; +// --- Building blocks: portable directory checkpoints ----------------------- +export { + PortableDirectoryCheckpointCaptureError, + PortableDirectoryCheckpointCodec, + PortableDirectoryCheckpointCorruptionError, + PortableDirectoryCheckpointFileSchema, + PortableDirectoryCheckpointPolicy, + PortableDirectoryCheckpointPolicyError, + PortableDirectoryCheckpointRestoreTargetError, + PortableDirectoryCheckpointService, +} from './core/checkpoint/index.js'; +export type { + PortableDirectoryCheckpointFile, + PortableDirectoryCheckpointLimits, + PortableDirectoryCheckpointPolicyOptions, +} from './core/checkpoint/index.js'; + // --- Building blocks: memory & knowledge ----------------------------------- export { buildMemoryDomainSystemContext } from './core/memory/domain-prompt.js'; export { diff --git a/src/core/checkpoint/index.ts b/src/core/checkpoint/index.ts new file mode 100644 index 00000000..ea75193e --- /dev/null +++ b/src/core/checkpoint/index.ts @@ -0,0 +1 @@ +export * from './portable-directory/index.js'; diff --git a/src/core/checkpoint/portable-directory/README.md b/src/core/checkpoint/portable-directory/README.md new file mode 100644 index 00000000..9cdfe32e --- /dev/null +++ b/src/core/checkpoint/portable-directory/README.md @@ -0,0 +1,83 @@ +# Portable Directory Checkpoints + +This module owns reusable local-filesystem mechanics for capturing and +restoring one explicitly selected directory. It lets durable domains share the +same path, symlink, integrity, resource-limit, and staged-restore guarantees +without pretending that they share one scope, manifest, store, or lifecycle. + +## Owns + +- Canonical POSIX-style relative file paths that cannot traverse the selected + root or become absolute on Windows. +- A base64, byte-length, and SHA-256 integrity envelope for regular files. +- An immutable `PortableDirectoryCheckpointPolicy` with an inclusion predicate + and explicit file-count, per-file byte, and total-byte limits. +- Deterministic, bounded capture that rejects symbolic links and selected + non-regular files. +- Validation of every restored file before the destination is touched. +- Restore through a sibling staging directory followed by one rename into an + absent or empty destination. + +## Does Not Own + +- Durable identity, scope derivation, generation or manifest schemas. +- Object-store keys, compare-and-swap, retention, or ambiguous-write recovery. +- When a conversation, heartbeat, or other workflow restores or checkpoints. +- Whether failed or cancelled executions should retain local changes. +- Product file conventions or a general workspace backup. + +Those decisions belong to the domain specialization and its host lifecycle. +For example, memory retains its released v1 generation, manifest, scope, store, +and error contracts while delegating directory mechanics here. A future +working-set specialization must define its own identity and durable store. + +## Policy Example + +```ts +import { + PortableDirectoryCheckpointPolicy, + PortableDirectoryCheckpointService, +} from '@heddleagent/runtime/advanced'; + +const policy = new PortableDirectoryCheckpointPolicy({ + includeFile: () => true, + limits: { + maxFileCount: 128, + maxFileBytes: 1024 * 1024, + maxTotalBytes: 16 * 1024 * 1024, + }, +}); + +const directory = new PortableDirectoryCheckpointService('/workspace/state', policy); +const files = await directory.capture(); +await directory.restore(files); +``` + +The example limits are illustrative, not framework defaults. Every +specialization must choose and document limits appropriate to its payload and +storage path. The caller must serialize capture/restore with mutations of the +same local working copy. Byte limits apply to decoded file content; generation +metadata and transport encoding remain the specialization's responsibility. + +Unsafe paths are rejected by default. The policy's `unsafePathBehavior: +'exclude'` mode exists for already-versioned domains whose released capture +behavior skipped such names; new domains should retain the fail-closed default. + +## Restore Safety + +`restore()` accepts only an absent or empty destination and never merges files. +It validates paths, inclusion, duplicates, ancestor conflicts, limits, base64, +lengths, and checksums before creating the staging directory. Files are written +with exclusive creation and mode `0600`, synchronized, and only then exposed by +renaming the complete staged directory. + +Capture opens files with `O_NOFOLLOW` where Node exposes it. Windows does not +provide that POSIX flag, so the service also compares the immediately preceding +`lstat` identity with the opened handle's `fstat` identity. This fallback keeps +the public contract cross-platform while still failing closed when the selected +leaf changes or resolves as a symbolic link. + +This is a local filesystem publication boundary, not distributed transaction +or execution fencing. A durable specialization still needs an immutable +generation plus an authoritative manifest compare-and-swap so a failed capture +or competing writer cannot replace the last committed generation. diff --git a/src/core/checkpoint/portable-directory/codec.ts b/src/core/checkpoint/portable-directory/codec.ts new file mode 100644 index 00000000..136fe4f6 --- /dev/null +++ b/src/core/checkpoint/portable-directory/codec.ts @@ -0,0 +1,74 @@ +import { createHash } from 'node:crypto'; +import { posix, win32 } from 'node:path'; +import { PortableDirectoryCheckpointCorruptionError } from './errors.js'; +import { + PortableDirectoryCheckpointFileSchema, + type PortableDirectoryCheckpointFile, +} from './schemas.js'; + +const WINDOWS_DRIVE_PREFIX = /^[A-Za-z]:/u; + +/** Owns the shared file envelope and path-independent integrity checks. */ +export class PortableDirectoryCheckpointCodec { + static isSafeRelativePath(path: string): boolean { + const segments = path.split('/'); + return Boolean(path) + && !path.includes('\\') + && !path.includes('\0') + && !posix.isAbsolute(path) + && !win32.isAbsolute(path) + && !WINDOWS_DRIVE_PREFIX.test(path) + && posix.normalize(path) === path + && segments.every(segment => Boolean(segment) && segment !== '.' && segment !== '..'); + } + + static createFile(path: string, content: Buffer): PortableDirectoryCheckpointFile { + if (!PortableDirectoryCheckpointCodec.isSafeRelativePath(path)) { + throw new PortableDirectoryCheckpointCorruptionError(`unsafe relative file path: ${path}`); + } + + return PortableDirectoryCheckpointFileSchema.parse({ + path, + contentBase64: content.toString('base64'), + byteLength: content.byteLength, + sha256: PortableDirectoryCheckpointCodec.sha256(content), + }); + } + + static decodeFile(value: unknown): Buffer { + const parsed = PortableDirectoryCheckpointFileSchema.safeParse(value); + if (!parsed.success) { + throw new PortableDirectoryCheckpointCorruptionError( + 'file does not match the supported integrity envelope', + { cause: parsed.error }, + ); + } + + const file = parsed.data; + if (!PortableDirectoryCheckpointCodec.isSafeRelativePath(file.path)) { + throw new PortableDirectoryCheckpointCorruptionError(`unsafe relative file path: ${file.path}`); + } + + const content = Buffer.from(file.contentBase64, 'base64'); + if (content.toString('base64') !== file.contentBase64) { + throw new PortableDirectoryCheckpointCorruptionError(`file has invalid base64 content: ${file.path}`); + } + if (content.byteLength !== file.byteLength) { + throw new PortableDirectoryCheckpointCorruptionError( + `file byte length does not match content: ${file.path}`, + ); + } + if (PortableDirectoryCheckpointCodec.sha256(content) !== file.sha256) { + throw new PortableDirectoryCheckpointCorruptionError(`file checksum does not match content: ${file.path}`); + } + return content; + } + + static comparePaths(left: string, right: string): number { + return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')); + } + + private static sha256(content: Buffer): string { + return createHash('sha256').update(content).digest('hex'); + } +} diff --git a/src/core/checkpoint/portable-directory/errors.ts b/src/core/checkpoint/portable-directory/errors.ts new file mode 100644 index 00000000..7463b96a --- /dev/null +++ b/src/core/checkpoint/portable-directory/errors.ts @@ -0,0 +1,43 @@ +/** Raised when a portable-directory policy contains unsafe or ambiguous limits. */ +export class PortableDirectoryCheckpointPolicyError extends Error { + readonly code = 'PORTABLE_DIRECTORY_CHECKPOINT_POLICY_ERROR'; + + constructor(readonly detail: string, options?: ErrorOptions) { + super(`Invalid portable directory checkpoint policy: ${detail}`, options); + this.name = 'PortableDirectoryCheckpointPolicyError'; + } +} + +/** Raised when a local directory cannot be captured within its selected policy. */ +export class PortableDirectoryCheckpointCaptureError extends Error { + readonly code = 'PORTABLE_DIRECTORY_CHECKPOINT_CAPTURE_ERROR'; + + constructor( + readonly directoryRoot: string, + readonly detail: string, + options?: ErrorOptions, + ) { + super(`Failed to capture portable directory checkpoint from ${directoryRoot}: ${detail}`, options); + this.name = 'PortableDirectoryCheckpointCaptureError'; + } +} + +/** Raised when checkpoint file metadata, content, or paths cannot be trusted. */ +export class PortableDirectoryCheckpointCorruptionError extends Error { + readonly code = 'PORTABLE_DIRECTORY_CHECKPOINT_CORRUPTION'; + + constructor(readonly detail: string, options?: ErrorOptions) { + super(`Invalid portable directory checkpoint: ${detail}`, options); + this.name = 'PortableDirectoryCheckpointCorruptionError'; + } +} + +/** Raised when restore would merge with or overwrite a local directory. */ +export class PortableDirectoryCheckpointRestoreTargetError extends Error { + readonly code = 'PORTABLE_DIRECTORY_CHECKPOINT_RESTORE_TARGET_NOT_EMPTY'; + + constructor(readonly directoryRoot: string) { + super(`Portable directory checkpoint restore requires an absent or empty directory: ${directoryRoot}`); + this.name = 'PortableDirectoryCheckpointRestoreTargetError'; + } +} diff --git a/src/core/checkpoint/portable-directory/index.ts b/src/core/checkpoint/portable-directory/index.ts new file mode 100644 index 00000000..1651b6d7 --- /dev/null +++ b/src/core/checkpoint/portable-directory/index.ts @@ -0,0 +1,17 @@ +export { PortableDirectoryCheckpointCodec } from './codec.js'; +export { + PortableDirectoryCheckpointCaptureError, + PortableDirectoryCheckpointCorruptionError, + PortableDirectoryCheckpointPolicyError, + PortableDirectoryCheckpointRestoreTargetError, +} from './errors.js'; +export { PortableDirectoryCheckpointPolicy } from './policy.js'; +export { + PortableDirectoryCheckpointFileSchema, +} from './schemas.js'; +export { PortableDirectoryCheckpointService } from './service.js'; +export type { PortableDirectoryCheckpointFile } from './schemas.js'; +export type { + PortableDirectoryCheckpointLimits, + PortableDirectoryCheckpointPolicyOptions, +} from './types.js'; diff --git a/src/core/checkpoint/portable-directory/policy.ts b/src/core/checkpoint/portable-directory/policy.ts new file mode 100644 index 00000000..46e4084c --- /dev/null +++ b/src/core/checkpoint/portable-directory/policy.ts @@ -0,0 +1,51 @@ +import { PortableDirectoryCheckpointCodec } from './codec.js'; +import { PortableDirectoryCheckpointPolicyError } from './errors.js'; +import type { + PortableDirectoryCheckpointLimits, + PortableDirectoryCheckpointPolicyOptions, +} from './types.js'; + +const LIMIT_NAMES: ReadonlyArray = [ + 'maxFileCount', + 'maxFileBytes', + 'maxTotalBytes', +]; + +/** + * Resolves one immutable selection and resource policy for directory capture + * and restore. Callers must choose every bound explicitly. + */ +export class PortableDirectoryCheckpointPolicy { + readonly limits: PortableDirectoryCheckpointLimits; + readonly unsafePathBehavior: 'reject' | 'exclude'; + private readonly includeFile: PortableDirectoryCheckpointPolicyOptions['includeFile']; + + constructor(options: PortableDirectoryCheckpointPolicyOptions) { + if (typeof options.includeFile !== 'function') { + throw new PortableDirectoryCheckpointPolicyError('includeFile must be a function'); + } + + for (const name of LIMIT_NAMES) { + const value = options.limits[name]; + if (!Number.isSafeInteger(value) || value < 0) { + throw new PortableDirectoryCheckpointPolicyError(`${name} must be a non-negative safe integer`); + } + } + if (options.unsafePathBehavior !== undefined + && options.unsafePathBehavior !== 'reject' + && options.unsafePathBehavior !== 'exclude') { + throw new PortableDirectoryCheckpointPolicyError( + 'unsafePathBehavior must be either reject or exclude', + ); + } + + this.limits = Object.freeze({ ...options.limits }); + this.unsafePathBehavior = options.unsafePathBehavior ?? 'reject'; + this.includeFile = options.includeFile; + } + + /** Returns false for both excluded files and paths that are not safely portable. */ + includes(path: string): boolean { + return PortableDirectoryCheckpointCodec.isSafeRelativePath(path) && this.includeFile(path); + } +} diff --git a/src/core/checkpoint/portable-directory/schemas.ts b/src/core/checkpoint/portable-directory/schemas.ts new file mode 100644 index 00000000..6538a7c4 --- /dev/null +++ b/src/core/checkpoint/portable-directory/schemas.ts @@ -0,0 +1,11 @@ +import { z } from 'zod'; + +/** Shared integrity envelope for one regular file in a portable directory. */ +export const PortableDirectoryCheckpointFileSchema = z.object({ + path: z.string().min(1), + contentBase64: z.string(), + byteLength: z.number().int().nonnegative(), + sha256: z.string().regex(/^[a-f0-9]{64}$/u), +}).strict(); + +export type PortableDirectoryCheckpointFile = z.infer; diff --git a/src/core/checkpoint/portable-directory/service.ts b/src/core/checkpoint/portable-directory/service.ts new file mode 100644 index 00000000..6f607e71 --- /dev/null +++ b/src/core/checkpoint/portable-directory/service.ts @@ -0,0 +1,377 @@ +import { constants } from 'node:fs'; +import { + lstat, + mkdir, + mkdtemp, + open, + readdir, + realpath, + rename, + rm, + rmdir, + type FileHandle, +} from 'node:fs/promises'; +import { basename, dirname, join, posix, relative, resolve, sep } from 'node:path'; +import { PortableDirectoryCheckpointCodec } from './codec.js'; +import { + PortableDirectoryCheckpointCaptureError, + PortableDirectoryCheckpointCorruptionError, + PortableDirectoryCheckpointRestoreTargetError, +} from './errors.js'; +import { PortableDirectoryCheckpointPolicy } from './policy.js'; +import type { PortableDirectoryCheckpointFile } from './schemas.js'; + +const READ_CHUNK_BYTES = 64 * 1024; + +type CaptureState = { + files: PortableDirectoryCheckpointFile[]; + totalBytes: number; +}; + +type PreparedFile = { + path: string; + content: Buffer; +}; + +type RestoreTargetState = + | { kind: 'absent' } + | { kind: 'empty-directory'; device: number; inode: number }; + +/** + * Owns bounded, symlink-free capture and staged restore for one selected local + * directory. Domain generations, manifests, scopes, stores, and lifecycle + * timing remain the responsibility of the specialization using this service. + */ +export class PortableDirectoryCheckpointService { + private readonly directoryRoot: string; + + constructor( + directoryRoot: string, + private readonly policy: PortableDirectoryCheckpointPolicy, + ) { + this.directoryRoot = resolve(directoryRoot); + } + + /** Captures selected regular files in canonical bytewise path order. */ + async capture(): Promise { + let canonicalRoot: string; + try { + canonicalRoot = await realpath(this.directoryRoot); + } catch (error) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + 'directory root does not exist', + { cause: error }, + ); + } + + let root: Awaited>; + try { + root = await lstat(canonicalRoot); + } catch (error) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + 'directory root could not be inspected safely', + { cause: error }, + ); + } + if (!root.isDirectory()) { + throw new PortableDirectoryCheckpointCaptureError(this.directoryRoot, 'directory root is not a directory'); + } + + const state: CaptureState = { files: [], totalBytes: 0 }; + try { + await this.walk(canonicalRoot, canonicalRoot, state); + return state.files.sort((left, right) => PortableDirectoryCheckpointCodec.comparePaths( + left.path, + right.path, + )); + } catch (error) { + if (error instanceof PortableDirectoryCheckpointCaptureError) { + throw error; + } + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + PortableDirectoryCheckpointService.errorMessage(error), + { cause: error }, + ); + } + } + + /** Validates every file before atomically presenting a staged working copy. */ + async restore(files: readonly PortableDirectoryCheckpointFile[]): Promise { + const preparedFiles = this.prepareFiles(files); + const targetState = await this.inspectRestoreTarget(); + const parent = dirname(this.directoryRoot); + await mkdir(parent, { recursive: true }); + const stagingRoot = await mkdtemp(join(parent, `.${basename(this.directoryRoot)}.restore-`)); + + try { + for (const file of preparedFiles) { + const targetPath = resolve(stagingRoot, ...file.path.split('/')); + await mkdir(dirname(targetPath), { recursive: true, mode: 0o700 }); + const handle = await open(targetPath, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL, 0o600); + try { + await handle.writeFile(file.content); + await handle.sync(); + } finally { + await handle.close(); + } + } + + await this.commitStagedRestore(targetState, stagingRoot); + return this.directoryRoot; + } finally { + await rm(stagingRoot, { recursive: true, force: true }); + } + } + + private async walk(root: string, directory: string, state: CaptureState): Promise { + const entries = (await readdir(directory, { withFileTypes: true })) + .sort((left, right) => PortableDirectoryCheckpointCodec.comparePaths(left.name, right.name)); + + for (const entry of entries) { + const path = join(directory, entry.name); + const portablePath = relative(root, path).split(sep).join('/'); + + if (entry.isSymbolicLink()) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `symbolic links are not portable directory state: ${portablePath}`, + ); + } + const safePath = PortableDirectoryCheckpointCodec.isSafeRelativePath(portablePath); + if (entry.isDirectory()) { + if (!safePath && this.policy.unsafePathBehavior === 'reject') { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `unsafe relative file path: ${portablePath}`, + ); + } + await this.walk(root, path, state); + continue; + } + if (!safePath) { + if (this.policy.unsafePathBehavior === 'exclude') { + continue; + } + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `unsafe relative file path: ${portablePath}`, + ); + } + if (!this.policy.includes(portablePath)) { + continue; + } + if (!entry.isFile()) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `selected path is not a regular file: ${portablePath}`, + ); + } + + this.assertCaptureFileCount(state.files.length + 1); + const content = await this.readBoundedFile(path, portablePath, state.totalBytes); + state.files.push(PortableDirectoryCheckpointCodec.createFile(portablePath, content)); + state.totalBytes += content.byteLength; + } + } + + private async readBoundedFile( + path: string, + portablePath: string, + capturedBytes: number, + ): Promise { + let pathIdentity: Awaited>; + try { + pathIdentity = await lstat(path); + } catch (error) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `selected file could not be inspected safely: ${portablePath}`, + { cause: error }, + ); + } + if (!pathIdentity.isFile() || pathIdentity.isSymbolicLink()) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + pathIdentity.isSymbolicLink() + ? `symbolic links are not portable directory state: ${portablePath}` + : `selected path is not a regular file: ${portablePath}`, + ); + } + + let handle: FileHandle; + try { + // O_NOFOLLOW is unavailable on Windows. The lstat/fstat identity check + // below is the portable fallback and also closes leaf replacement races + // on platforms that do support the flag. + handle = await open(path, constants.O_RDONLY | (constants.O_NOFOLLOW ?? 0)); + } catch (error) { + const detail = PortableDirectoryCheckpointService.isErrorWithCode(error, 'ELOOP') + ? `symbolic links are not portable directory state: ${portablePath}` + : `selected file could not be opened safely: ${portablePath}`; + throw new PortableDirectoryCheckpointCaptureError(this.directoryRoot, detail, { cause: error }); + } + + try { + const file = await handle.stat(); + if (!file.isFile() || file.dev !== pathIdentity.dev || file.ino !== pathIdentity.ino) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `selected file changed identity during capture: ${portablePath}`, + ); + } + this.assertCaptureBytes(portablePath, file.size, capturedBytes); + + const chunks: Buffer[] = []; + let byteLength = 0; + let position = 0; + while (true) { + const remaining = Math.min( + this.policy.limits.maxFileBytes - byteLength, + this.policy.limits.maxTotalBytes - capturedBytes - byteLength, + ); + const readLength = remaining >= READ_CHUNK_BYTES ? READ_CHUNK_BYTES : remaining + 1; + const chunk = Buffer.allocUnsafe(Math.max(1, readLength)); + const result = await handle.read(chunk, 0, chunk.byteLength, position); + if (result.bytesRead === 0) { + break; + } + + byteLength += result.bytesRead; + this.assertCaptureBytes(portablePath, byteLength, capturedBytes); + chunks.push(Buffer.from(chunk.subarray(0, result.bytesRead))); + position += result.bytesRead; + } + return Buffer.concat(chunks, byteLength); + } finally { + await handle.close(); + } + } + + private prepareFiles(files: readonly PortableDirectoryCheckpointFile[]): PreparedFile[] { + if (files.length > this.policy.limits.maxFileCount) { + throw new PortableDirectoryCheckpointCorruptionError( + `file count ${files.length} exceeds limit ${this.policy.limits.maxFileCount}`, + ); + } + + const paths = new Set(); + let totalBytes = 0; + const prepared = files.map(file => { + const content = PortableDirectoryCheckpointCodec.decodeFile(file); + if (!this.policy.includes(file.path)) { + throw new PortableDirectoryCheckpointCorruptionError(`file is excluded by policy: ${file.path}`); + } + if (paths.has(file.path)) { + throw new PortableDirectoryCheckpointCorruptionError(`duplicate file path: ${file.path}`); + } + if (content.byteLength > this.policy.limits.maxFileBytes) { + throw new PortableDirectoryCheckpointCorruptionError( + `file ${file.path} has ${content.byteLength} bytes, exceeding limit ${this.policy.limits.maxFileBytes}`, + ); + } + + const nextTotal = totalBytes + content.byteLength; + if (!Number.isSafeInteger(nextTotal) || nextTotal > this.policy.limits.maxTotalBytes) { + throw new PortableDirectoryCheckpointCorruptionError( + `total bytes ${nextTotal} exceed limit ${this.policy.limits.maxTotalBytes}`, + ); + } + + paths.add(file.path); + totalBytes = nextTotal; + return { path: file.path, content }; + }); + + for (const path of paths) { + let ancestor = posix.dirname(path); + while (ancestor !== '.') { + if (paths.has(ancestor)) { + throw new PortableDirectoryCheckpointCorruptionError( + `file path conflicts with descendant: ${ancestor}`, + ); + } + ancestor = posix.dirname(ancestor); + } + } + + return prepared.sort((left, right) => PortableDirectoryCheckpointCodec.comparePaths(left.path, right.path)); + } + + private async inspectRestoreTarget(): Promise { + try { + const target = await lstat(this.directoryRoot); + if (!target.isDirectory() || target.isSymbolicLink() || (await readdir(this.directoryRoot)).length > 0) { + throw new PortableDirectoryCheckpointRestoreTargetError(this.directoryRoot); + } + return { kind: 'empty-directory', device: target.dev, inode: target.ino }; + } catch (error) { + if (PortableDirectoryCheckpointService.isErrorWithCode(error, 'ENOENT')) { + return { kind: 'absent' }; + } + throw error; + } + } + + private async commitStagedRestore(state: RestoreTargetState, stagingRoot: string): Promise { + if (state.kind === 'absent') { + try { + await lstat(this.directoryRoot); + throw new PortableDirectoryCheckpointRestoreTargetError(this.directoryRoot); + } catch (error) { + if (!PortableDirectoryCheckpointService.isErrorWithCode(error, 'ENOENT')) { + throw error; + } + } + } else { + const target = await lstat(this.directoryRoot); + const targetChanged = !target.isDirectory() + || target.isSymbolicLink() + || target.dev !== state.device + || target.ino !== state.inode + || (await readdir(this.directoryRoot)).length > 0; + if (targetChanged) { + throw new PortableDirectoryCheckpointRestoreTargetError(this.directoryRoot); + } + await rmdir(this.directoryRoot); + } + + await rename(stagingRoot, this.directoryRoot); + } + + private assertCaptureFileCount(fileCount: number): void { + if (fileCount > this.policy.limits.maxFileCount) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `file count ${fileCount} exceeds limit ${this.policy.limits.maxFileCount}`, + ); + } + } + + private assertCaptureBytes(path: string, fileBytes: number, capturedBytes: number): void { + if (!Number.isSafeInteger(fileBytes) || fileBytes > this.policy.limits.maxFileBytes) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `file ${path} has ${fileBytes} bytes, exceeding limit ${this.policy.limits.maxFileBytes}`, + ); + } + + const totalBytes = capturedBytes + fileBytes; + if (!Number.isSafeInteger(totalBytes) || totalBytes > this.policy.limits.maxTotalBytes) { + throw new PortableDirectoryCheckpointCaptureError( + this.directoryRoot, + `total bytes ${totalBytes} exceed limit ${this.policy.limits.maxTotalBytes}`, + ); + } + } + + private static errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error); + } + + private static isErrorWithCode(error: unknown, code: string): error is Error & { code: string } { + return error instanceof Error && 'code' in error && error.code === code; + } +} diff --git a/src/core/checkpoint/portable-directory/types.ts b/src/core/checkpoint/portable-directory/types.ts new file mode 100644 index 00000000..633c7038 --- /dev/null +++ b/src/core/checkpoint/portable-directory/types.ts @@ -0,0 +1,12 @@ +export type PortableDirectoryCheckpointLimits = Readonly<{ + maxFileCount: number; + maxFileBytes: number; + maxTotalBytes: number; +}>; + +export type PortableDirectoryCheckpointPolicyOptions = { + limits: PortableDirectoryCheckpointLimits; + includeFile: (relativePath: string) => boolean; + /** Defaults to `reject`; `exclude` exists only for versioned compatibility policies. */ + unsafePathBehavior?: 'reject' | 'exclude'; +}; diff --git a/src/core/memory/README.md b/src/core/memory/README.md index 9a138768..8ddd166d 100644 --- a/src/core/memory/README.md +++ b/src/core/memory/README.md @@ -40,8 +40,11 @@ guidance. so the same subject and owner resolve the same memory after a fresh session. - `checkpoint/`: provider-neutral checkpoint schemas, deterministic memory-file capture, integrity validation, restore-before-use, and a manifest-last store - contract. The host supplies the durable store and invokes checkpoints at - stable memory boundaries; see its [`README`](checkpoint/README.md). + contract. It preserves the released memory-v1 wire/storage contract while + delegating reusable path, integrity, and staged-directory mechanics to + `src/core/checkpoint/portable-directory`. The host supplies the durable store + and invokes checkpoints at stable memory boundaries; see its + [`README`](checkpoint/README.md). - `catalog.ts`: `MemoryCatalogService` owns catalog bootstrap, root catalog loading, startup system-context assembly, and required catalog shape. - `domain-prompt.ts`: memory-specific system context. diff --git a/src/core/memory/checkpoint/README.md b/src/core/memory/checkpoint/README.md index 46996b82..89378e05 100644 --- a/src/core/memory/checkpoint/README.md +++ b/src/core/memory/checkpoint/README.md @@ -6,6 +6,13 @@ object store, or a purpose-built implementation of `MemoryCheckpointStore`. The provider does not redefine Heddle's file selection, encoding, validation, or restore semantics. +The filesystem mechanics delegate to +[`src/core/checkpoint/portable-directory`](../../checkpoint/portable-directory/README.md). +Memory keeps its released v1 scope, generation, manifest, store, error, and +serialized-byte contracts. Its compatibility policy does not add new default +file-count or byte rejection; future bounded directory domains must select +their own explicit operational limits. + ## Owns - A versioned generation and committed-manifest schema. @@ -15,6 +22,10 @@ or restore semantics. - Manifest compare-and-swap expectations for concurrent writers. - Explicit deletion of the authoritative manifest. +The shared portable-directory service owns canonical relative paths, regular +file integrity envelopes, symlink rejection, bounded reads, and staged local +restore. It does not own memory's allowlist or durable generation semantics. + ## Portable Files A checkpoint contains: diff --git a/src/core/memory/checkpoint/codec.ts b/src/core/memory/checkpoint/codec.ts index a30be3e3..2cde9c6d 100644 --- a/src/core/memory/checkpoint/codec.ts +++ b/src/core/memory/checkpoint/codec.ts @@ -1,6 +1,10 @@ import { createHash } from 'node:crypto'; -import { posix } from 'node:path'; +import { + PortableDirectoryCheckpointCodec, + PortableDirectoryCheckpointCorruptionError, +} from '../../checkpoint/portable-directory/index.js'; import type { MemoryScopeId } from '../scope.js'; +import { memoryCheckpointDirectoryPolicy } from './directory-policy.js'; import { MemoryCheckpointCorruptionError } from './errors.js'; import { MemoryCheckpointFileSchema, @@ -12,11 +16,6 @@ import { type MemoryCheckpointManifest, } from './schemas.js'; -const PORTABLE_MAINTENANCE_FILES = new Set([ - '_maintenance/candidates.jsonl', - '_maintenance/runs.jsonl', -]); - type CreateMemoryCheckpointGenerationInput = { scopeId: MemoryScopeId; generationId: MemoryCheckpointGenerationId; @@ -35,24 +34,11 @@ type CreateMemoryCheckpointManifestInput = { */ export class MemoryCheckpointCodec { static isPortablePath(path: string): boolean { - if (!MemoryCheckpointCodec.isCanonicalRelativePath(path)) { - return false; - } - - if (PORTABLE_MAINTENANCE_FILES.has(path)) { - return true; - } - - return !path.startsWith('_maintenance/') && path.endsWith('.md'); + return memoryCheckpointDirectoryPolicy.includes(path); } static createFile(path: string, content: Buffer): MemoryCheckpointFile { - return MemoryCheckpointFileSchema.parse({ - path, - contentBase64: content.toString('base64'), - byteLength: content.byteLength, - sha256: MemoryCheckpointCodec.sha256(content), - }); + return MemoryCheckpointFileSchema.parse(PortableDirectoryCheckpointCodec.createFile(path, content)); } static createGeneration(input: CreateMemoryCheckpointGenerationInput): MemoryCheckpointGeneration { @@ -60,7 +46,10 @@ export class MemoryCheckpointCodec { kind: 'heddle-memory-checkpoint-generation', schemaVersion: 1, ...input, - files: [...input.files].sort((left, right) => MemoryCheckpointCodec.comparePaths(left.path, right.path)), + files: [...input.files].sort((left, right) => PortableDirectoryCheckpointCodec.comparePaths( + left.path, + right.path, + )), }); MemoryCheckpointCodec.validateFiles(generation.scopeId, generation.files); @@ -158,17 +147,14 @@ export class MemoryCheckpointCodec { throw new MemoryCheckpointCorruptionError(scopeId, `path is not portable memory state: ${file.path}`); } - const content = Buffer.from(file.contentBase64, 'base64'); - if (content.toString('base64') !== file.contentBase64) { - throw new MemoryCheckpointCorruptionError(scopeId, `file has invalid base64 content: ${file.path}`); - } - if (content.byteLength !== file.byteLength) { - throw new MemoryCheckpointCorruptionError(scopeId, `file byte length does not match content: ${file.path}`); - } - if (MemoryCheckpointCodec.sha256(content) !== file.sha256) { - throw new MemoryCheckpointCorruptionError(scopeId, `file checksum does not match content: ${file.path}`); + try { + return PortableDirectoryCheckpointCodec.decodeFile(file); + } catch (error) { + if (error instanceof PortableDirectoryCheckpointCorruptionError) { + throw new MemoryCheckpointCorruptionError(scopeId, error.detail, { cause: error }); + } + throw error; } - return content; } private static validateFiles(scopeId: MemoryScopeId, files: MemoryCheckpointFile[]): void { @@ -180,7 +166,7 @@ export class MemoryCheckpointCodec { if (paths.has(file.path)) { throw new MemoryCheckpointCorruptionError(scopeId, `duplicate file path: ${file.path}`); } - if (priorPath && MemoryCheckpointCodec.comparePaths(priorPath, file.path) >= 0) { + if (priorPath && PortableDirectoryCheckpointCodec.comparePaths(priorPath, file.path) >= 0) { throw new MemoryCheckpointCorruptionError(scopeId, 'generation files are not in canonical path order'); } paths.add(file.path); @@ -188,20 +174,6 @@ export class MemoryCheckpointCodec { } } - private static isCanonicalRelativePath(path: string): boolean { - return Boolean(path) - && !path.includes('\\') - && !path.includes('\0') - && !posix.isAbsolute(path) - && posix.normalize(path) === path - && path !== '.' - && !path.startsWith('../'); - } - - private static comparePaths(left: string, right: string): number { - return Buffer.compare(Buffer.from(left, 'utf8'), Buffer.from(right, 'utf8')); - } - private static generationSha256(generation: MemoryCheckpointGeneration): string { return MemoryCheckpointCodec.sha256(Buffer.from(MemoryCheckpointCodec.serializeGeneration(generation), 'utf8')); } diff --git a/src/core/memory/checkpoint/directory-policy.ts b/src/core/memory/checkpoint/directory-policy.ts new file mode 100644 index 00000000..ddccf3c5 --- /dev/null +++ b/src/core/memory/checkpoint/directory-policy.ts @@ -0,0 +1,22 @@ +import { PortableDirectoryCheckpointPolicy } from '../../checkpoint/portable-directory/index.js'; + +const PORTABLE_MAINTENANCE_FILES = new Set([ + '_maintenance/candidates.jsonl', + '_maintenance/runs.jsonl', +]); + +/** + * Memory v1 predates portable-directory resource limits. Safe-integer ceilings + * preserve its accepted behavior while routing through the shared mechanism; + * tightening them requires an explicit opt-in or a versioned memory contract. + */ +export const memoryCheckpointDirectoryPolicy = new PortableDirectoryCheckpointPolicy({ + includeFile: path => PORTABLE_MAINTENANCE_FILES.has(path) + || (!path.startsWith('_maintenance/') && path.endsWith('.md')), + unsafePathBehavior: 'exclude', + limits: { + maxFileCount: Number.MAX_SAFE_INTEGER, + maxFileBytes: Number.MAX_SAFE_INTEGER, + maxTotalBytes: Number.MAX_SAFE_INTEGER, + }, +}); diff --git a/src/core/memory/checkpoint/schemas.ts b/src/core/memory/checkpoint/schemas.ts index 83d5e3f5..b2aa33ec 100644 --- a/src/core/memory/checkpoint/schemas.ts +++ b/src/core/memory/checkpoint/schemas.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import { PortableDirectoryCheckpointFileSchema } from '../../checkpoint/portable-directory/index.js'; import { MemoryScopeIdSchema } from '../scope.js'; export const MEMORY_CHECKPOINT_SCHEMA_VERSION = 1 as const; @@ -10,12 +11,9 @@ export const MemoryCheckpointGenerationIdSchema = z .regex(/^[A-Za-z0-9][A-Za-z0-9._-]*$/u) .brand('MemoryCheckpointGenerationId'); -export const MemoryCheckpointFileSchema = z.object({ - path: z.string().min(1), - contentBase64: z.string(), - byteLength: z.number().int().nonnegative(), - sha256: z.string().regex(/^[a-f0-9]{64}$/u), -}).strict(); +// Alias the shared file envelope so the released memory-v1 wire shape and key +// order stay byte-for-byte stable. +export const MemoryCheckpointFileSchema = PortableDirectoryCheckpointFileSchema; export const MemoryCheckpointGenerationSchema = z.object({ kind: z.literal('heddle-memory-checkpoint-generation'), diff --git a/src/core/memory/checkpoint/service.ts b/src/core/memory/checkpoint/service.ts index bb6bec28..3796a6e5 100644 --- a/src/core/memory/checkpoint/service.ts +++ b/src/core/memory/checkpoint/service.ts @@ -1,19 +1,13 @@ import { randomUUID } from 'node:crypto'; import { - lstat, - mkdir, - mkdtemp, - readFile, - readdir, - realpath, - rename, - rm, - rmdir, - writeFile, -} from 'node:fs/promises'; -import { basename, dirname, join, relative, resolve, sep } from 'node:path'; + PortableDirectoryCheckpointCaptureError, + PortableDirectoryCheckpointCorruptionError, + PortableDirectoryCheckpointRestoreTargetError, + PortableDirectoryCheckpointService, +} from '../../checkpoint/portable-directory/index.js'; import type { MemoryScopeId } from '../scope.js'; import { MemoryCheckpointCodec } from './codec.js'; +import { memoryCheckpointDirectoryPolicy } from './directory-policy.js'; import { MemoryCheckpointCaptureError, MemoryCheckpointCorruptionError, @@ -38,6 +32,7 @@ import type { export class MemoryCheckpointService { private readonly now: () => Date; private readonly createGenerationId: NonNullable; + private readonly directory: PortableDirectoryCheckpointService; constructor( private readonly memoryRoot: string, @@ -47,6 +42,7 @@ export class MemoryCheckpointService { this.now = options.now ?? (() => new Date()); this.createGenerationId = options.createGenerationId ?? (() => MemoryCheckpointGenerationIdSchema.parse(`memory-generation-v1-${randomUUID()}`)); + this.directory = new PortableDirectoryCheckpointService(this.memoryRoot, memoryCheckpointDirectoryPolicy); } /** @@ -114,9 +110,7 @@ export class MemoryCheckpointService { return { status: 'absent' }; } - const memoryRoot = resolve(this.memoryRoot); - await this.prepareRestoreTarget(memoryRoot); - await this.restoreGeneration(memoryRoot, checkpoint); + const memoryRoot = await this.restoreGeneration(checkpoint); return { status: 'restored', memoryRoot, @@ -146,95 +140,31 @@ export class MemoryCheckpointService { } private async captureFiles(): Promise { - const configuredRoot = resolve(this.memoryRoot); - let memoryRoot: string; try { - memoryRoot = await realpath(configuredRoot); + return await this.directory.capture(); } catch (error) { - throw new MemoryCheckpointCaptureError(configuredRoot, 'memory root does not exist', { cause: error }); - } - - const files: MemoryCheckpointFile[] = []; - await this.walkMemoryFiles(memoryRoot, memoryRoot, files); - return files; - } - - private async walkMemoryFiles( - memoryRoot: string, - directory: string, - files: MemoryCheckpointFile[], - ): Promise { - const entries = (await readdir(directory, { withFileTypes: true })) - .sort((left, right) => Buffer.compare(Buffer.from(left.name), Buffer.from(right.name))); - - for (const entry of entries) { - const path = join(directory, entry.name); - const portablePath = relative(memoryRoot, path).split(sep).join('/'); - - if (entry.isSymbolicLink()) { - throw new MemoryCheckpointCaptureError( - memoryRoot, - `symbolic links are not portable memory state: ${portablePath}`, - ); - } - if (entry.isDirectory()) { - await this.walkMemoryFiles(memoryRoot, path, files); - continue; - } - if (!MemoryCheckpointCodec.isPortablePath(portablePath)) { - continue; + if (error instanceof PortableDirectoryCheckpointCaptureError) { + throw new MemoryCheckpointCaptureError(error.directoryRoot, error.detail, { cause: error }); } - if (!entry.isFile()) { - throw new MemoryCheckpointCaptureError( - memoryRoot, - `portable memory path is not a regular file: ${portablePath}`, - ); - } - - files.push(MemoryCheckpointCodec.createFile(portablePath, await readFile(path))); + throw error; } } - private async prepareRestoreTarget(memoryRoot: string): Promise { + private async restoreGeneration(checkpoint: MemoryCheckpointBundle): Promise { try { - const target = await lstat(memoryRoot); - if (!target.isDirectory() || target.isSymbolicLink()) { - throw new MemoryCheckpointRestoreTargetError(memoryRoot); - } - if ((await readdir(memoryRoot)).length > 0) { - throw new MemoryCheckpointRestoreTargetError(memoryRoot); - } - await rmdir(memoryRoot); + return await this.directory.restore(checkpoint.generation.files); } catch (error) { - if (MemoryCheckpointService.isErrorWithCode(error, 'ENOENT')) { - return; + if (error instanceof PortableDirectoryCheckpointRestoreTargetError) { + throw new MemoryCheckpointRestoreTargetError(error.directoryRoot); } - throw error; - } - } - - private async restoreGeneration(memoryRoot: string, checkpoint: MemoryCheckpointBundle): Promise { - const parent = dirname(memoryRoot); - await mkdir(parent, { recursive: true }); - const stagingRoot = await mkdtemp(join(parent, `.${basename(memoryRoot)}.restore-`)); - - try { - for (const file of checkpoint.generation.files) { - const targetPath = resolve(stagingRoot, ...file.path.split('/')); - await mkdir(dirname(targetPath), { recursive: true }); - await writeFile( - targetPath, - MemoryCheckpointCodec.decodeFile(checkpoint.manifest.scopeId, file), - { flag: 'wx', mode: 0o600 }, + if (error instanceof PortableDirectoryCheckpointCorruptionError) { + throw new MemoryCheckpointCorruptionError( + checkpoint.manifest.scopeId, + error.detail, + { cause: error }, ); } - await rename(stagingRoot, memoryRoot); - } finally { - await rm(stagingRoot, { recursive: true, force: true }); + throw error; } } - - private static isErrorWithCode(error: unknown, code: string): error is Error & { code: string } { - return error instanceof Error && 'code' in error && error.code === code; - } }