diff --git a/AGENTS.md b/AGENTS.md index b16e43fe1..96394efb0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -200,6 +200,7 @@ import { EntityFetcherProvider, EntityGrid } from "@metaobjectsdev/tanstack"; - **Codegen substrate**: ts-poet for greenfield emit, ts-morph for in-place edits, Biome for format pass, `git merge-file --diff3` for hand-edit-preserving regen. - **Runtime substrate**: Kysely for TS (user-provided connection, async-only). - **Migration substrate**: Postgres + SQLite for TS v0.3. +- **Metadata location**: resolved via `resolveCollection()` (`@metaobjectsdev/sdk`) — the single authority. `metaobjects/` is the **default value of `sources`** and nothing else: no other module, command or user-facing message may assert that a directory of that name exists or is where metadata lives. Exactly six sites may name it — `sdk/src/metadata-files.ts` (`DEFAULT_METADATA_DIR`, its single definition), `sdk/src/sources.ts` (`DEFAULT_SOURCES`, **the** default), `sdk/src/collection.ts` (inside `resolveCollection`, *applying* that default), `sdk/src/index.ts` (the barrel re-export of the constant, no use), `cli/src/commands/init.ts` (the scaffolder **writing** the layout), and `sdk/src/agent-docs/body.ts` (the agent-docs prose `meta init` scaffolds beside that layout). Enforced by `sdk/test/no-hardcoded-metadata-dir.test.ts`, whose allowlist demands a written reason per entry. See [docs/features/metadata-sources.md](docs/features/metadata-sources.md). ## Explicitly out of scope @@ -224,6 +225,8 @@ import { EntityFetcherProvider, EntityGrid } from "@metaobjectsdev/tanstack"; **Default convention**: one file per domain concept under `metaobjects/`. Multiple objects per file when they share a domain. Projections (`source.dbView`) live inline with their base entity. +`metaobjects/` is the **default value** of `sources` in `.metaobjects/config.json` — never a requirement. A project declaring `sources` explicitly can point anywhere (and need not have such a directory at all); `"sources": []`, which is what `meta init` scaffolds, takes the default. + ``` project-root/ ├── metaobjects/ # VISIBLE — entity declarations diff --git a/CHANGELOG.md b/CHANGELOG.md index 481468c2d..75a8ce566 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -91,6 +91,59 @@ the local release path published `docs-site` ahead of `metadata` and `render`, t packages it depends on. The tier is declared now, and an undeclared one is an error instead of an accidental position. +### Metadata source resolution — adopter-visible changes + +`.metaobjects/config.json` gains `sources`, `scope` and `migrate.scope`, and every +command resolves where metadata lives through one authority instead of reading a +hardcoded `metaobjects/` directory. A project with one config at its root, no +`sources` and no `scope` resolves the same files, generates the same code and emits +the same migrations. Three changes are visible even to that project. Adopter guide: +[`docs/features/metadata-sources.md`](docs/features/metadata-sources.md#upgrading). + +- **The workspace `extends:` walk is retired.** `loadMemory` used to have a second, + undocumented way of finding metadata: a `package.meta.json` declaring `extends:` + dependencies, inside a discoverable workspace (`pnpm-workspace.yaml` or + `package.json` `workspaces`), pulled in each peer package's `metaobjects/` + directory first, in topological order. Every CLI read path now resolves through + `sources`, which does no such walk. It fails LOUDLY — `ERR_UNRESOLVED_SUPER` + naming the target it cannot find, never a half-resolved model — and the + replacement is an explicit `{ "path": "../shared-model/metaobjects" }` source, + which works in any layout and needs no topological ordering. +- **`.metaobjects/config.json` rejects unknown keys.** `ConfigSchema` is `.strict()` + at every level, so a key that was previously stripped in silence is now a load + error naming the key. Silently dropping a key means the setting you wrote does not + exist: `{ "migrate": { "scopee": [...] } }` used to mean *unscoped*, governing + every table in a database you were trying to share. +- **`ExpectedView.fqn` is required.** On the public `@metaobjectsdev/codegen-ts` + export, the declaring object's fully-qualified name is no longer optional — + `migrate.scope` decides ownership on that name, and a view arriving without one + cannot be scoped at all. `buildProjectionViews` already supplies it; only + hand-built `ExpectedView` values need the field added. +- **`meta export` output order changed, and `_pending/` is excluded.** `export` now + serializes the file set `resolveCollection` resolved rather than scanning a + directory through `DirectorySource`, so siblings emit files-before-subdirectories + (the overlay-safe order the loader has always been given) instead of a flat + basename sort, and staged `_pending/` files — skipped by every other read path — + are no longer exported. The canonical JSON content is unchanged; a committed + export diffed against a fresh one shows a reordering. +- **The migrations directory follows the project root.** `.metaobjects/migrations` + and the schema snapshot resolve from the directory whose `.metaobjects/config.json` + governs the run, found by walking up from the working directory. `meta migrate + apply-pending` and `--rollback` load no metadata and previously used the working + directory unconditionally, so a subdirectory holding a ledger but no config of its + own now replays the project root's history. `migrate` says so out loud when the + resolved directory differs from `/.metaobjects/migrations` and that local + directory exists; `--out-dir` overrides, and giving the subdirectory its own + `.metaobjects/config.json` makes it a project root. +- **A project boundary is a `.metaobjects/config.json` — a bare `metaobjects/` + directory is not one.** Discovery walks up for a config and stops at nothing + else short of the `.git` boundary, so a command run inside a nested directory + that holds metadata but declares no config of its own resolves the nearest + ancestor config — adopting its `sources` and `outDir`. `metaobjects/` is the + default *value* of `sources`, so a directory of that name says nothing about + whether a project lives there. If a subdirectory should own its metadata, give + it a config: `meta init` writes one, and a `"sources": []` config is enough to + claim the directory and take the default. ## [0.23.2] — npm `0.23.2` · PyPI `0.23.2` · NuGet `0.23.2` · Maven `7.23.2` diff --git a/docs/CONFORMANCE.md b/docs/CONFORMANCE.md index 09ae7efaa..dafb26fd6 100644 --- a/docs/CONFORMANCE.md +++ b/docs/CONFORMANCE.md @@ -1,6 +1,6 @@ # Conformance coverage -The MetaObjects standard ships **19 shared conformance corpora** under +The MetaObjects standard ships **20 shared conformance corpora** under [`fixtures/`](../fixtures/). Every port runs every corpus that is *applicable to it* and asserts the same expected behaviour against the same fixtures. **This page is the inverse index**: fixture → feature doc + per-port pass status, and it is the @@ -42,6 +42,7 @@ regenerate with `ls -d fixtures//*/ | wc -l`. | [`fixtures/template-output-render-conformance/`](../fixtures/template-output-render-conformance/) | 5 | ✓ | ✓ | ✓ | ✓ | ✓ | | [`fixtures/generator-registry-conformance/`](../fixtures/generator-registry-conformance/) | 1 canonical manifest | ✓ | ✓ | ✓ | ✓ | ✓ | | [`fixtures/provider-composition-conformance/`](../fixtures/provider-composition-conformance/) | 9 (5 error-shape + 4 compose-load) | ✓ | ✓ | — (JVM registry via Java) | ✓ | ✓ | +| [`fixtures/scope-conformance/`](../fixtures/scope-conformance/) | 10 cases | ✓ (reference implementation) | — | — | — | — | | [`fixtures/agent-context-conformance/`](../fixtures/agent-context-conformance/) | 4 | ✓ (the emitter is TS-owned) | — | — | — | — | | [`fixtures/metamodel-docs/`](../fixtures/metamodel-docs/) | 1 | ✓ (docs emit is TS-owned) | — | — | — | — | @@ -138,10 +139,33 @@ inheritance), `m2m/` (3), `jsonb/` (2, typed value-object columns) and Kotlin, C#, Python — run it in BOTH lanes: a hand-rolled reference server and the port's own GENERATED API artifact booted over HTTP. +### `fixtures/scope-conformance/` (10 cases) + +All 10 cases → [features/metadata-sources.md](features/metadata-sources.md) (the +`scope` pattern grammar). The corpus is file-shaped: one committed `cases.json`, +read directly by every port's runner, with no per-port fixture and no ledger. + +It pins the semantics of a consumer's `include`/`exclude` scope over +fully-qualified names — **`*` matches within one `::` segment and never crosses +it; a segment that is exactly `**` matches one or more whole segments (so +`acme::**` does not match the bare `acme`); every other character is literal, +regex metacharacters included; an absent or empty `include` means everything; +multiple `include` patterns are a union and `exclude` is applied after it; and +matching is case-sensitive.** These are exactly the rules four independent +implementations would otherwise each get slightly wrong — the failure mode that +produced the cross-port `LIKE`/`ILIKE` divergence fixed in 0.21.6. + +**TypeScript is the only port with a runner today.** The reference implementation is +[`server/typescript/packages/sdk/src/scope.ts`](../server/typescript/packages/sdk/src/scope.ts) +(`compilePattern` / `compileScope` / `matchesScope`), and the corpus was authored +against it. Java, Kotlin, C# and Python have no runner yet; when each gains one, this corpus is +what it implements against — it exists now precisely so those four land on one +grammar rather than four. + ## Orphaned fixtures (tested but not yet documented) -The fixtures in the six corpora mapped above (metamodel 255 + yaml 15 + verify 31 -+ render 15 + persistence 33 + api-contract 41) each map to a feature doc. None +The fixtures in the seven corpora mapped above (metamodel 255 + yaml 15 + verify 31 ++ render 15 + persistence 33 + api-contract 41 + scope 10) each map to a feature doc. None are orphaned today. The remaining corpora in the totals table gate tooling contracts (registry manifests, provider composition, agent context, docs emit) rather than user-facing metamodel behaviour, so they have no feature-doc row. diff --git a/docs/README.md b/docs/README.md index bf86035a8..7903aad34 100644 --- a/docs/README.md +++ b/docs/README.md @@ -37,6 +37,7 @@ docs/ │ ├── downstream-metadata-decisions.md # guidance for adopters extending the metamodel │ ├── generated-mutations.md # generated POST/PATCH mutation surface │ ├── image-upload.md # view.image form control (TS-web) +│ ├── metadata-sources.md # where metadata comes from: sources, scope, discovery │ └── own-your-codegen.md # scaffold-and-own generator ownership (ADR-0034) └── ports/ # one file per language/framework port ├── typescript.md @@ -58,6 +59,7 @@ this tree is documentation, not the source of truth. | Understand what `object.entity`, `source.rdb`, `template.prompt` mean | [`features/`](features/) | | Compare what TS vs Java vs Kotlin vs C# vs Python emit for the same metadata | any [`features/*.md`](features/) — every feature shows all five ports side-by-side | | Author metadata in YAML instead of JSON | [`features/yaml-authoring.md`](features/yaml-authoring.md) | +| Point the toolchain at metadata that lives somewhere other than `metaobjects/`, or scope what a project generates and migrates | [`features/metadata-sources.md`](features/metadata-sources.md) | | Record what the system is supposed to do, and stop agents reviving retired features | [`features/requirements.md`](features/requirements.md) | | Wire prompt construction (FR-004) | [`features/templates-and-payloads.md`](features/templates-and-payloads.md) | | Share a metadata shape across multiple instances (abstracts, `extends:`) | [`features/abstracts-and-inheritance.md`](features/abstracts-and-inheritance.md) | diff --git a/docs/features/metadata-sources.md b/docs/features/metadata-sources.md new file mode 100644 index 000000000..930a63044 --- /dev/null +++ b/docs/features/metadata-sources.md @@ -0,0 +1,626 @@ +# Metadata sources, scope, and discovery + +**Where does my metadata come from?** From the `sources` set in +`.metaobjects/config.json`. When that key is absent or empty, `sources` takes its +default value — the `metaobjects/` directory sitting beside the `.metaobjects/` +folder that holds the config. + +**`metaobjects/` is that default value, and nothing more.** It is not a requirement, +and it is not a convention any part of the toolchain is allowed to assume: a project +that declares `sources` may put its metadata anywhere — a sibling module, a shared +model repository, a single file — and need not have a directory of that name at all. +Every command answers "where is the metadata?" by reading the config, so pointing +`sources` elsewhere moves *all* of them together. Outside `resolveCollection`'s own +default-applies check, nothing greps for the directory or assumes it exists — and the +only place that names it in a message is the `meta init` scaffolded agent-docs prose, +which is documentation content, not a resolution path. + +**How do I point it somewhere else?** Declare it: + +```json +{ + "schema_version": 1, + "sources": [ + { "path": "../model/src/main/resources/metadata" }, + { "path": "metaobjects" } + ] +} +``` + +`meta gen`, `meta migrate`, `meta verify`, `meta docs` and `meta export` all read +exactly that set. A `path` is read **in place and never installed** or copied. + +**Nothing breaks if you do nothing.** `meta init` has always scaffolded +`"sources": []`, so every existing project takes the default and resolves the same +files it always did. + +**Port support.** `sources` / `scope` / `migrate.scope` are read by the **Node +`meta` CLI** today. The Java, Kotlin, C# and Python CLIs still take their metadata +location their own way (a Maven `` element, a positional directory, a +`metadata` config key). The cross-port pattern grammar is already pinned by +[`fixtures/scope-conformance/`](../../fixtures/scope-conformance/); wiring the other +four CLIs to the same config file is future work, not yet planned or scheduled. + +--- + +## `sources` — a set, not an ordered list + +`sources` is an array for authoring convenience, but it is **specified as a set**. +Reordering it cannot change what resolves: `resolveSources` walks the entries in +**content order** rather than declared order, so both the resolved file order and — +when two entries overlap on the same file — the entry recorded as its provenance are +decided by content, never by which was declared first. + +That is a real guarantee rather than a stylistic claim, because the loader does not +need an order either — it derives overlay precedence from the files themselves +(see [Order independence](#order-independence-is-three-layers) below). + +Consequences worth knowing: + +- Two entries may overlap. A file reached by two `path` entries is loaded once. +- There is no cycle detection, because a set union cannot have a cycle. +- A `path` that does not exist is an **error** (`ERR_SOURCE_UNRESOLVED`), never a + silent skip. Only the *default* is allowed to be absent, and only then to produce + the friendlier `ERR_COLLECTION_NOT_FOUND`. + +### The `path` kind + +```jsonc +"sources": [ + { "path": "metaobjects" }, // a directory, relative to this config + { "path": "../shared-model/metadata" }, // a sibling module — read in place + { "path": "vendor/model/meta.catalog.json" } // a single file +] +``` + +- A relative `path` resolves **against the directory holding the declaring + `.metaobjects/` folder**, never against the ambient working directory. Moving + where you run the command from cannot change what resolves. +- An absolute `path` is taken as-is. +- A directory is walked **recursively**. A file counts as metadata when its + extension is `.json`, `.yaml` or `.yml`, matched case-insensitively. +- A `_pending/` directory is skipped at any depth (it holds proposed, unpromoted + records). +- Symlinked subdirectories are followed, matching the loader's own directory walk. + +### `resource` and `package` are declared but not resolved + +```jsonc +"sources": [ + { "resource": "acme/model" }, // reserved — JVM classpath resource root + { "package": "@acme/common-model" } // reserved — a published metadata package +] +``` + +Both parse (the config shape is fixed now so a later phase slots in without a config +migration) and both throw `ERR_SOURCE_KIND_UNSUPPORTED` when the toolchain tries to +resolve them. A misspelled key such as `{ "pathh": "model" }` is a **load error**, +not a silently ignored extra: `.metaobjects/config.json` is validated strictly, top +level and every nested block. + +--- + +## Discovery — nearest ancestor, stopping at the repository boundary + +Running a CLI inside an app should find that app's configuration, not the repo +root's. + +1. Start at the working directory (`--cwd` / `-C` moves the starting point). +2. Walk **up**, looking for `.metaobjects/config.json`. The **first one found wins** + — a config in a subdirectory beats one in an ancestor. +3. **Stop after examining a directory containing `.git`.** A checkout can never + silently adopt a parent checkout's configuration. The config check runs before + the boundary check within each directory, so a config at the repository root — + sharing its directory with `.git` — is still reachable from any subdirectory. +4. If nothing is found, the starting directory is used with the default `sources`. + +That config file is the **only** thing the walk looks for. A directory holding a +`metaobjects/` directory but no config is not a project root — see +[A project boundary is a `.metaobjects/config.json`](#a-project-boundary-is-a-metaobjectsconfigjson) +under Upgrading. + +Collections are **never auto-discovered**. Nothing globs the tree for directories +that merely look like metadata homes; a collection exists only where a config names +one. + +**A config that exists but fails to load propagates its error.** Malformed JSON or a +schema violation is the author's mistake, and it fails loudly — it does not fall +through to the default and quietly generate from a stale `metaobjects/`. + +**`ERR_COLLECTION_NOT_FOUND`** is raised only when *both* have failed: no `sources` +were declared anywhere up the walk, **and** no default `metaobjects/` directory +exists either. + +--- + +## `scope` — an output filter over fully-qualified names + +```jsonc +"scope": { + "include": ["acme::blog::**", "acme::common::*"], + "exclude": ["acme::blog::internal::**"] +} +``` + +**The collection always loads in full. Scope filters output, never input.** This is +deliberate and not an optimization left on the table: a partial file list can fail to +load outright when an `extends` target is in a file that was filtered away, so an +author would have to hand-maintain a transitive closure. Loading everything is +closure-complete by definition, and the filter is applied where the toolchain emits. + +### Pattern grammar + +Patterns match a node's fully-qualified name — `::`, e.g. +`acme::blog::Author`. An object declared with no package has a bare name as its FQN. + +| Rule | Example | +|---|---| +| `::` separates segments | `acme::blog::Author` is three segments | +| `*` matches any run of characters **within one segment**, never crossing `::` | `acme::blog::Author*` matches `acme::blog::AuthorDraft`, not `acme::blog::x::AuthorDraft` | +| A segment that is exactly `**` matches **one or more** whole segments | `acme::**` matches `acme::Author` and `acme::a::b::Author`, but **not** the bare `acme` | +| `**` mid-pattern still requires at least one segment | `acme::**::Author` does **not** match `acme::Author` | +| Every other character is **literal**, regex metacharacters included | `acme::v1.0::*` matches a segment literally named `v1.0` | +| Absent or empty `include` means **everything** | `{ "exclude": ["acme::blog::internal::**"] }` narrows the default | +| Multiple `include` patterns are a **union** | a name matches if any one matches | +| `exclude` applies **after** `include` | an excluded name stays excluded no matter which `include` admitted it | +| Matching is **case-sensitive** | `acme::Author` does not match `acme::author` | + +An unparseable pattern is an error (`ERR_SCOPE_PATTERN_INVALID`), never a silent +non-match. Empty patterns, empty segments, and a malformed separator (an odd run of +`:`) all fail loudly at load. + +A common first mistake: `acme::*` matches only objects **one** segment below `acme`. +For a package tree, you want `acme::**`. + +### Where `scope` applies + +| Command | Scoped by top-level `scope`? | +|---|---| +| `meta gen` | **Yes** — an object is generated only when its FQN is in scope | +| `meta verify --codegen` | **Yes** — it regenerates under the same scope, so a scoped `gen` cannot be reported as drift | +| `meta docs` | No | +| `meta export` | No | +| `meta migrate`, `meta verify --db` | No — those take [`migrate.scope`](#migratescope--who-owns-which-tables) instead | + +`docs` and `export` are **inspection surfaces over the loaded collection**, not code +emitters. Scoping them would make the tools you reach for to answer "what is +actually in this model?" answer a narrower question than the one you asked. + +`meta gen ` arguments **intersect** with scope: both must pass. If a scope +leaves nothing to generate, `gen` says so and names the scope as the reason rather +than blaming the entity filter. + +### The one sharp edge + +An in-scope object may reference an out-of-scope one — an FK target, a +relationship `@objectRef`, a projection's base. The reference resolves perfectly at +load time (everything loaded), but the code emitted for the in-scope object names a +symbol that was never generated here. + +This is left to fail loudly rather than silently auto-widening the scope: you +declared the scope precisely because something else owns those objects. The failure +is an unresolved import — a plain compiler error at build time, not a surprise at +runtime. + +### Per-generator scope is not phase 1 + +The TypeScript-only per-generator **`filter` function** in +`metaobjects.config.ts` is unchanged and remains supported as an escape hatch: + +```ts +entityFile({ filter: (e) => e.name !== "Legacy" }) +``` + +It is deliberately not the thing a cross-port feature depends on — a JavaScript +predicate cannot be written in a `pom.xml`, a Python config, or a C# CLI flag, and +no conformance corpus can gate it. Package patterns are strings and port unchanged +to all five config surfaces. A declarative per-generator `scope` key is deferred. + +--- + +## `migrate.scope` — who owns which tables + +A database is often shared: this consumer owns one package tree's tables, another +tool owns the rest. Without a declaration, `meta migrate` treats every table it does +not model as a table to **drop**. + +```jsonc +"migrate": { + "outDir": "./.metaobjects/migrations", + "databaseUrl": "postgres://localhost:5432/acme", + "dialect": "postgres", + "scope": ["acme::billing::**"] +} +``` + +`migrate.scope` is a **plain array of include patterns** — the same grammar as +top-level `scope`, with no `exclude` arm. A migration run is scoped to what it +governs, not filtered down from "everything". + +Tables and views whose declaring object falls outside the scope are **neither created +nor dropped**. That takes two suppressions, and the toolchain does both: the objects +leave the *expected* schema, and their physical names are suppressed on the *actual* +side too. Doing only the first would be strictly worse than doing nothing — every +out-of-scope table that already exists would become a proposed `DROP TABLE`. + +- **`meta migrate`** prints what it left alone: `N object(s) out-of-scope (outside + migrate.scope, governed elsewhere)`, naming the tables. Without that line, "no + changes" and "no changes to the half of the model this run governs" read + identically. +- **`meta verify --db`** reports out-of-scope objects as out-of-scope rather than as + drift, and applies the same narrowing to the committed schema snapshot. +- **`meta migrate baseline` is deliberately unscoped.** A `--from-db` baseline + records a starting point read out of the database; it has no metadata provenance to + scope by. An out-of-scope table sitting in that snapshot is harmless — the diff is + scoped on every subsequent run. +- A table or view with **no recorded provenance is kept**. Scope decides on the + declaring object's FQN, and an object whose FQN is unknown was never proven to be + anyone else's. + +### Put the `migrate` block where the ledger lives + +**Whoever holds `.metaobjects/migrations/` and the schema snapshot owns the schema.** +A repository with six codegen consumers over one database has at most one schema +owner; if each declared a `migrate` block you would get six partial migrations, which +is worse than having none. + +This is also mechanically required today: `migrate.scope` is read from the +**discovered** config, but the rest of the `migrate` block (`outDir`, +`databaseUrl`, `dialect`, `allow`, `d1`) is read from `.metaobjects/config.json` in +the directory you run the command in. Run `meta migrate` from the directory that +holds both the config and the ledger, or pass `--cwd` to point at it. + +`meta verify --db` may run from any consumer — it reports rather than writes. + +--- + +## Vendoring — airgapped and hermetic builds + +There is no separate vendoring mechanism, and none is needed. Because a `path` +source is **read in place and never installed**, vendoring is: + +1. Copy the dependency's metadata into a directory in your repository. +2. Point a `path` at it. +3. Commit it. + +```jsonc +{ + "schema_version": 1, + "sources": [ + { "path": "vendor/acme-common-model" }, + { "path": "metaobjects" } + ] +} +``` + +The build now resolves entirely from committed files, with no network access and no +resolution step that could produce a different answer tomorrow than it did today — +the `go mod vendor` property, obtained by declaring a directory. + +Because sources are a set, the vendored entry needs no particular position. If the +vendored tree and your own both declare the same node, ordinary overlay merge rules +apply — see [`loaders.md`](loaders.md). + +--- + +## A worked polyglot example + +A repository where a Maven module owns the model, two Node consumers generate from +it, and exactly one of them owns the database. + +``` +acme-platform/ +├── .git/ +├── model/ # Maven module — the model, no CLI config +│ └── src/main/resources/metadata/ +│ ├── meta.common.json # package acme::common +│ ├── meta.billing.json # package acme::billing +│ └── meta.blog.json # package acme::blog +├── services/billing/ # Node consumer — SCHEMA OWNER +│ └── .metaobjects/ +│ ├── config.json +│ └── migrations/ # the ledger lives here +└── apps/web/ # Node consumer — codegen only + └── .metaobjects/ + └── config.json +``` + +`services/billing/.metaobjects/config.json` — reaches the Maven module's resource +directory as a plain path, generates only the billing tree, and owns the billing +tables: + +```json +{ + "schema_version": 1, + "sources": [ + { "path": "../../model/src/main/resources/metadata" } + ], + "scope": { + "include": ["acme::billing::**", "acme::common::**"] + }, + "migrate": { + "outDir": "./.metaobjects/migrations", + "databaseUrl": "postgres://localhost:5432/acme", + "dialect": "postgres", + "scope": ["acme::billing::**"] + } +} +``` + +`apps/web/.metaobjects/config.json` — same model, different slice, **no `migrate` +block** because it does not own the schema: + +```json +{ + "schema_version": 1, + "sources": [ + { "path": "../../model/src/main/resources/metadata" } + ], + "scope": { + "include": ["acme::blog::**", "acme::common::**"], + "exclude": ["acme::blog::internal::**"] + } +} +``` + +What this buys: + +- **No symlinks and no copied files.** The Maven module stays the single home of the + metadata; both Node consumers read it in place. The Java build is untouched — it + keeps using its own Maven configuration. +- **Running `meta gen` in `apps/web/` finds `apps/web`'s config**, because discovery + walks up from the working directory and takes the nearest one. It never reaches + `services/billing`, and the `.git` at `acme-platform/` stops it from escaping the + checkout. +- **`meta migrate` from `services/billing/`** proposes changes to `acme::billing` + tables only. The `acme::blog` tables — owned by a different tool sharing the same + database — are neither created nor dropped, and are reported as out-of-scope. +- **`meta verify --db` from either consumer** reports honestly: the web app sees the + billing tables as out-of-scope, not as drift. + +- **Running a command at `acme-platform/` itself fails**, rather than guessing. There + is no config there, `.git` stops the walk, and no default `metaobjects/` directory + exists — so `ERR_COLLECTION_NOT_FOUND` names both halves. Run from a consumer, or + point `--cwd` at one. + +To make this repository build with no network access, copy +`model/src/main/resources/metadata` to `vendor/model/` in each consumer and change +one line per config. + +--- + +## Order independence is three layers + +Worth knowing precisely, because the layers are easy to conflate and they are not +redundant. + +1. **`resolveSources` canonicalizes.** It processes the entries in content order, so + in production the loader never sees a permuted file list at all. Canonical is not + the same as "sorted": within one directory entry the files keep the walk order the + toolchain has always used — the files at a level, then that level's + subdirectories, depth-first — because declaration order survives into generated + output (a barrel's export list, the shared `enums.ts`, `meta docs` page order, + `meta export`'s sibling order). Flat-sorting the paths would silently reorder any + project holding a subdirectory whose name sorts before a sibling file. +2. **The loader resolves content order-independently.** Overlay-only sources are + stable-partitioned to merge last, so an overlay reaching a base declared in + another file resolves the same regardless of which arrived first. +3. **Sibling order of unrelated top-level nodes still follows load order**, and that + is *not* a contract — the canonical serializer only ever promised attribute-key + alphabetization. Do not expect byte-identical whole-tree serialization across + permuted loader inputs. + +Layers 1 and 2 are pinned by +[`server/typescript/packages/sdk/test/order-independence.test.ts`](../../server/typescript/packages/sdk/test/order-independence.test.ts); +the per-level walk order layer 1 preserves is pinned by +[`server/typescript/packages/sdk/test/source-order.test.ts`](../../server/typescript/packages/sdk/test/source-order.test.ts). + +--- + +## Errors + +| Code | Raised when | +|---|---| +| `ERR_SOURCE_UNRESOLVED` | A declared `path` source does not exist on disk | +| `ERR_SOURCE_KIND_UNSUPPORTED` | A `resource` or `package` source was declared; this toolchain resolves `path` only | +| `ERR_SCOPE_PATTERN_INVALID` | A scope pattern is empty, has an empty segment, or has a malformed `::` separator | +| `ERR_COLLECTION_NOT_FOUND` | No `sources` were declared **and** no default `metaobjects/` directory exists | + +A schema violation in `.metaobjects/config.json` itself (an unknown key, a wrong +type) surfaces as the config load error and stops the command. + +--- + +## Upgrading + +A project with one config at its root, no `sources` and no `scope` resolves the same +files it always did and generates the same code. Six changes are still worth knowing +about before you upgrade. + +### A project boundary is a `.metaobjects/config.json` + +Discovery walks up for that file and stops at nothing else. A directory holding a +`metaobjects/` directory but no config of its own is **not** a project root, so a +command run inside it resolves the nearest ancestor config — including that config's +`sources`, which may point somewhere neither directory contains. + +This is the rule rather than a caveat: `metaobjects/` is the default *value* of +`sources`, so a directory of that name says nothing about whether a project lives +there. Treating it as a second marker would put a second answer to "where does +metadata live?" back into the toolchain, and would be silently wrong for every +project that declares `sources` elsewhere. + +If a subdirectory should own its metadata, give it a config — `meta init` writes +one, and a `"sources": []` config is enough to claim the directory and take the +default. + +### The workspace `extends:` walk is retired + +`loadMemory` used to have a second, hidden way of finding metadata: if the project +carried a `package.meta.json` declaring `extends:` dependencies **and** a workspace +could be discovered (`pnpm-workspace.yaml`, or `package.json` `workspaces`), it +walked that dependency graph and loaded each peer package's `metaobjects/` directory +first, in topological order. + +Every read path — `loadMemory` itself included — now resolves its files through +`sources`, which does no such walk. Two ways to find metadata, one of them implicit +and reachable only from a particular repository layout, is precisely the divergence +this feature exists to remove — and one of them was undocumented. + +**It fails loudly, not silently.** A model that depended on a peer package's +declarations now fails to load with `ERR_UNRESOLVED_SUPER` naming the `extends:` +target it cannot find; nothing generates from a half-resolved model. + +**A caller invoking `loadMemory` directly, not through the CLI, changes too.** With no +`options.files`, `loadMemory` now resolves its own file list via `resolveCollection` — +so a project with nothing to resolve (no declared `sources`, no default +`metaobjects/`) used to reject with a plain `Error("cannot read metadata directory +…")` and now rejects with a `ParseError` carrying `code: "ERR_COLLECTION_NOT_FOUND"` +(`sdk/src/collection.ts:158-163`) — the same structured code every other command +reports. A `catch` matching the old message text stops matching, silently. + +**The replacement is a declared source**, which is explicit and works in any layout, +workspace or not: + +```json +{ + "schema_version": 1, + "sources": [ + { "path": "../shared-model/metaobjects" }, + { "path": "metaobjects" } + ] +} +``` + +Order does not matter (see [`sources` — a set, not an ordered +list](#sources--a-set-not-an-ordered-list)), so there is no topological ordering to +reproduce. + +### `.metaobjects/config.json` rejects unknown keys + +`ConfigSchema` is `.strict()` at every level — the top level, `sources` entries, +`scope`, `migrate`, and `migrate.d1`. A key that used to be silently dropped is now +a load error that stops the command. + +This is deliberate and it is the whole point: a stripped key means the setting you +wrote does not exist, and the command runs as if you had never written it. +`{ "migrate": { "scopee": [...] } }` used to mean *unscoped* — governing every table +in a database you were trying to share. + +If a command starts failing on a config that used to load, the message names the +unrecognized key; fix the spelling or delete the key. + +### `ExpectedView.fqn` is required + +`ExpectedView` is a public `@metaobjectsdev/codegen-ts` export. Its `fqn` — the +declaring object's fully-qualified name — is now **required** rather than optional, +because `migrate.scope` decides on that name and a view arriving without one cannot +be scoped at all. Code that builds `ExpectedView` values by hand (the normal path, +`buildProjectionViews`, already supplies it) has to add the field. + +### `meta export` walks one tree, and skips `_pending/` + +`meta export` used to scan a directory through `DirectorySource`; it now serializes +the file set `resolveCollection` resolved, which is the same walk every other command +uses. Two things change in its output, for every project — declared `sources` or not: + +- **Sibling order is files-before-subdirectories**, not a flat basename sort. A + directory whose subdirectory name sorts before a sibling file (`admin/` before + `user.json`) therefore emits in a different order than it used to. This is the + order the loader has always been given, and it is the overlay-safe one — a base + file must load before an overlay nested under it — so `export` and the loader now + agree instead of disagreeing. +- **`_pending/` is excluded.** Those files are staged, not active metadata; every + other read path already skipped them, and `export` was the one that did not. + +The canonical JSON *content* is unchanged. If you diff a committed export against a +freshly generated one, expect a reordering and the loss of any `_pending/` entries. + +### The migrations directory follows the project root, not the shell + +`.metaobjects/migrations` (and the schema snapshot beside it) is resolved from the +directory whose `.metaobjects/config.json` governs the run, discovered by walking up +from the working directory. It used to come from the working directory +unconditionally for `meta migrate apply-pending` and `meta migrate --rollback`, which +load no metadata at all. + +The ledger belongs with the config that declares it, so this is the intended +behaviour — but it moves the ledger for one layout: a subdirectory holding +`.metaobjects/migrations` with **no** `.metaobjects/config.json` of its own, sitting +under a project root that has both. Running `apply-pending` there now replays the +root's history. + +It is not silent. When the resolved migrations directory differs from +`/.metaobjects/migrations` **and** that local directory exists, the command +prints which one it is using and how to override: + +``` +migrate: using the migrations directory /repo/.metaobjects/migrations, not the +/repo/apps/api/.metaobjects/migrations in this working directory — the ledger belongs +to the project root that declares it. Pass --out-dir … to use the local one. +``` + +`--out-dir` (or `migrate.outDir` in the config) selects the directory explicitly, and +adding a `.metaobjects/config.json` to that subdirectory makes it a project root in +its own right — which is what a directory holding its own ledger almost always wants. + +--- + +## What is deferred + +Phase 1 ships the spine. Explicitly **not** built yet, so you do not go looking: + +- **`resource` sources** (JVM classpath roots) and **`package` sources** (a published + metadata package) — declared in the config shape, rejected at resolution. +- **`url` sources** and **named `collection` references**. +- **Per-generator declarative `scope`.** The TypeScript `filter` function is the + escape hatch and is unchanged. +- **The other four ports' CLIs reading `.metaobjects/config.json`.** The pattern + grammar is corpus-gated so they cannot diverge when they land. +- **Database and other runtime metadata sources.** Ruled a runtime-metadata concern + rather than a build-time one. + +Design rationale and the full phase plan: +[`docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md`](../superpowers/specs/2026-08-17-metadata-source-resolution-design.md). + +--- + +## Verified by + +**Cross-port pattern semantics** + +- [`fixtures/scope-conformance/`](../../fixtures/scope-conformance/) — 10 cases + pinning `*` / `**`, include-union, exclude-after-include, literal metacharacters, + and case sensitivity. TypeScript runs it today; the other four ports have no runner + yet. See [`CONFORMANCE.md`](../CONFORMANCE.md). + +**TypeScript gates** + +- `server/typescript/packages/sdk/test/scope.test.ts` — the pattern engine +- `server/typescript/packages/sdk/test/scope-conformance.test.ts` — the corpus runner +- `server/typescript/packages/sdk/test/sources.test.ts` — `path` resolution, `_pending` + exclusion, unsupported kinds +- `server/typescript/packages/sdk/test/discovery.test.ts` — nearest-ancestor walk and + the `.git` boundary +- `server/typescript/packages/sdk/test/collection.test.ts` — `resolveCollection` + precedence, the `metaobjects/` default, and error propagation +- `server/typescript/packages/sdk/test/order-independence.test.ts` — layers 1 and 2 above +- `server/typescript/packages/sdk/test/dogfood-examples.test.ts` — a consumer reaching + a real committed metadata tree, with scope evaluated over the FQNs the loader + actually produced +- `server/typescript/packages/cli/test/collection-routing.test.ts` — every command + routing through `resolveCollection` +- `server/typescript/packages/cli/test/migrate-scope.test.ts` and + `server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts` — + both-sided `migrate.scope` suppression +- `server/typescript/packages/codegen-ts/test/run-gen.test.ts` — scope intersecting + the entity filter at the `gen` choke point + +## See also + +- [`loaders.md`](loaders.md) — how the resolved file set is merged +- [`cli.md`](cli.md) — the locked CLI architecture (ADR-0015) and which port owns what +- [`migrations-and-drift.md`](migrations-and-drift.md) — `meta migrate` and + `meta verify --db` +- [`own-your-codegen.md`](own-your-codegen.md) — generator ownership and the `filter` + escape hatch diff --git a/docs/features/requirements.md b/docs/features/requirements.md index 36b9be081..a76c4b3db 100644 --- a/docs/features/requirements.md +++ b/docs/features/requirements.md @@ -27,7 +27,7 @@ disproof to the thing being resurrected, in one line. ## Declaring one -Requirements live in `metaobjects/` beside the entities they describe: +Requirements live beside the entities they describe (by default in `metaobjects/`): ```jsonc { "metadata.root": { diff --git a/docs/superpowers/plans/2026-08-17-metadata-source-resolution-phase1-ts.md b/docs/superpowers/plans/2026-08-17-metadata-source-resolution-phase1-ts.md new file mode 100644 index 000000000..d509a0a5b --- /dev/null +++ b/docs/superpowers/plans/2026-08-17-metadata-source-resolution-phase1-ts.md @@ -0,0 +1,1686 @@ +# Metadata Source Resolution — Phase 1 (TypeScript) Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Make `sources` in `.metaobjects/config.json` the single authority on where metadata lives, with package-pattern scoping at output and nearest-ancestor discovery, so a consumer can point at a metadata tree elsewhere in the repo and take the slice it needs. + +**Architecture:** Sources are an unordered **set** of tagged specs resolved to a canonically-sorted file list (the loader already derives whatever order it needs, so declared order carries no information). Scope is a package-pattern include/exclude filter applied at **output**, never to input — the collection always loads in full, which makes every scope closure-complete by construction. One new `resolveCollection()` entry point replaces nine hardcoded `metaobjects/` reads. + +**Tech Stack:** TypeScript (ESM), Bun test runner, Zod for config schema, existing `@metaobjectsdev/metadata` loader and canonical serializer. + +**Spec:** `docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md` + +## Global Constraints + +- **Named constants for metamodel strings — always.** Import `PACKAGE_SEPARATOR` from `@metaobjectsdev/metadata/constants`; never inline `"::"`. +- **No `any`.** Use `unknown` and narrow. +- **Never `instanceof` a metadata node from another package** — use the exported guards (`isMetaObject`, `isMetaField`, …). Two physical copies of `metadata` in one process make `instanceof` silently false. +- **Never call `own*()` accessors** (ADR-0039). Resolving/effective accessors are the default. +- **Backward compatibility is absolute:** a project with one config at the root, no `sources`, and no `scope` must produce **byte-identical** output to today. Every task that touches a read path must prove this. +- **Public repository.** No private project names, no absolute home paths (`/home//…`) in code, tests, fixtures, or commit messages. +- **Run tests scoped:** `cd server/typescript && bun test packages/` — never a bare `bun test` at the repo root. +- Package separator is `::`. `*` matches any characters within one segment; a segment that is exactly `**` matches one or more segments. + +--- + +## File Structure + +**New — `server/typescript/packages/sdk/src/`** +- `scope.ts` — package-pattern compile + match. Pure, no I/O. The cross-port semantic core. +- `sources.ts` — `SourceSpec` union → canonically-sorted absolute file list. All filesystem I/O for source resolution. +- `discovery.ts` — nearest-ancestor config lookup. Filesystem walk only. +- `collection.ts` — `resolveCollection()`: the single authority composing the three above. + +**New — tests** +- `packages/sdk/test/scope.test.ts`, `sources.test.ts`, `discovery.test.ts`, `collection.test.ts` +- `packages/sdk/test/order-independence.test.ts` — the linchpin gate +- `packages/sdk/test/scope-conformance.test.ts` — runs the shared corpus + +**New — cross-port fixture** +- `fixtures/scope-conformance/cases.json`, `README.md` + +**Modified** +- `packages/sdk/src/config.ts` — widen `sources`, add `scope`, add `migrate.scope` +- `packages/sdk/src/memory.ts` — `loadMemory` accepts a resolved file list +- `packages/sdk/src/index.ts` — export the new surface +- `packages/cli/src/commands/{gen,docs,export,migrate}.ts` — route reads through `resolveCollection()` +- `packages/cli/src/index.ts:275` — the "is this a MetaObjects project?" probe +- `packages/cli/src/lib/detect-stack.ts` — route + nested-symlink fix +- `packages/metadata/src/errors.ts` — register new error codes + +**Deliberately unchanged** +- `packages/cli/src/commands/init.ts` — scaffolding writes the default directory. This is the one place the `"metaobjects"` literal belongs. + +--- + +## Task 1: Scope pattern engine + +**Files:** +- Create: `server/typescript/packages/sdk/src/scope.ts` +- Test: `server/typescript/packages/sdk/test/scope.test.ts` + +**Interfaces:** +- Consumes: `PACKAGE_SEPARATOR` from `@metaobjectsdev/metadata/constants` +- Produces: + - `interface Scope { readonly include?: readonly string[]; readonly exclude?: readonly string[] }` + - `interface CompiledScope { readonly include: readonly RegExp[]; readonly exclude: readonly RegExp[] }` + - `function compileScope(scope: Scope): CompiledScope` — throws `Error` whose message starts `ERR_SCOPE_PATTERN_INVALID` on an empty or malformed pattern + - `function matchesScope(fqn: string, compiled: CompiledScope): boolean` + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/sdk/test/scope.test.ts +import { describe, test, expect } from "bun:test"; +import { compileScope, matchesScope, type Scope } from "../src/scope.js"; + +const match = (fqn: string, scope: Scope) => matchesScope(fqn, compileScope(scope)); + +describe("compileScope / matchesScope", () => { + test("empty include matches everything", () => { + expect(match("acme::commerce::Order", {})).toBe(true); + }); + + test("* matches exactly one segment", () => { + const s: Scope = { include: ["acme::*"] }; + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::commerce::Order", s)).toBe(false); + }); + + test("** matches one or more segments", () => { + const s: Scope = { include: ["acme::**"] }; + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme", s)).toBe(false); + expect(match("other::Order", s)).toBe(false); + }); + + test("* within a segment matches a partial name but never crosses ::", () => { + const s: Scope = { include: ["acme::Order*"] }; + expect(match("acme::OrderLine", s)).toBe(true); + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::deep::OrderLine", s)).toBe(false); + }); + + test("exclude is applied after include", () => { + const s: Scope = { include: ["acme::**"], exclude: ["acme::internal::**"] }; + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme::internal::Secret", s)).toBe(false); + }); + + test("exclude alone narrows the implicit match-everything", () => { + const s: Scope = { exclude: ["acme::internal::**"] }; + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme::internal::Secret", s)).toBe(false); + }); + + test("a bare name with no package is matchable", () => { + expect(match("Order", { include: ["Order"] })).toBe(true); + expect(match("Order", { include: ["*"] })).toBe(true); + }); + + test("regex metacharacters in a pattern are literal", () => { + expect(match("acme::Order.v2", { include: ["acme::Order.v2"] })).toBe(true); + expect(match("acme::OrderXv2", { include: ["acme::Order.v2"] })).toBe(false); + }); + + test("an empty pattern is ERR_SCOPE_PATTERN_INVALID", () => { + expect(() => compileScope({ include: [""] })).toThrow(/ERR_SCOPE_PATTERN_INVALID/); + }); + + test("an empty segment is ERR_SCOPE_PATTERN_INVALID", () => { + expect(() => compileScope({ include: ["acme::::Order"] })).toThrow(/ERR_SCOPE_PATTERN_INVALID/); + }); +}); +``` + +- [ ] **Step 2: Run test to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/scope.test.ts` +Expected: FAIL — `Cannot find module '../src/scope.js'` + +- [ ] **Step 3: Write minimal implementation** + +```ts +// server/typescript/packages/sdk/src/scope.ts +import { PACKAGE_SEPARATOR } from "@metaobjectsdev/metadata/constants"; + +/** A consumer-side output filter over fully-qualified node names. */ +export interface Scope { + /** Absent or empty means "everything". */ + readonly include?: readonly string[]; + /** Applied after `include`. */ + readonly exclude?: readonly string[]; +} + +export interface CompiledScope { + readonly include: readonly RegExp[]; + readonly exclude: readonly RegExp[]; +} + +/** One package segment: any run of characters containing no separator char. */ +const SEGMENT = "[^:]+"; +/** One or more segments, separator-joined — the `**` expansion. */ +const SEGMENTS = `${SEGMENT}(?:${PACKAGE_SEPARATOR}${SEGMENT})*`; + +function escapeLiteral(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** Compile one segment. `**` spans segments; `*` never crosses a separator. */ +function compileSegment(segment: string, pattern: string): string { + if (segment.length === 0) { + throw new Error( + `ERR_SCOPE_PATTERN_INVALID: empty segment in scope pattern "${pattern}"`, + ); + } + if (segment === "**") return `(?:${SEGMENTS})`; + // `*` inside a segment matches any characters except the separator char. + return segment.split("*").map(escapeLiteral).join("[^:]*"); +} + +export function compilePattern(pattern: string): RegExp { + if (pattern.length === 0) { + throw new Error("ERR_SCOPE_PATTERN_INVALID: scope pattern must not be empty"); + } + const body = pattern + .split(PACKAGE_SEPARATOR) + .map((segment) => compileSegment(segment, pattern)) + .join(PACKAGE_SEPARATOR); + return new RegExp(`^${body}$`); +} + +export function compileScope(scope: Scope): CompiledScope { + return { + include: (scope.include ?? []).map(compilePattern), + exclude: (scope.exclude ?? []).map(compilePattern), + }; +} + +/** True when `fqn` is inside the scope. An empty `include` means everything. */ +export function matchesScope(fqn: string, compiled: CompiledScope): boolean { + const included = + compiled.include.length === 0 || compiled.include.some((re) => re.test(fqn)); + if (!included) return false; + return !compiled.exclude.some((re) => re.test(fqn)); +} +``` + +- [ ] **Step 4: Run test to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/scope.test.ts` +Expected: PASS — 10 tests + +- [ ] **Step 5: Typecheck** + +Run: `cd server/typescript && bun run --filter '@metaobjectsdev/sdk' typecheck` +Expected: no errors + +- [ ] **Step 6: Commit** + +```bash +git add server/typescript/packages/sdk/src/scope.ts server/typescript/packages/sdk/test/scope.test.ts +git commit -m "feat(sdk): package-pattern scope engine (* = one segment, ** = one or more)" +``` + +--- + +## Task 2: Scope-pattern conformance corpus + +Pins the semantics cross-port so `*` and `**` cannot come to mean five different things — the failure mode that produced the cross-port `LIKE`/`ILIKE` divergence. + +**Files:** +- Create: `fixtures/scope-conformance/cases.json` +- Create: `fixtures/scope-conformance/README.md` +- Test: `server/typescript/packages/sdk/test/scope-conformance.test.ts` + +**Interfaces:** +- Consumes: `compileScope` / `matchesScope` from Task 1 +- Produces: the corpus contract — `{ cases: Array<{ name: string; scope: {include?: string[]; exclude?: string[]}; expect: Array<{ fqn: string; matches: boolean }> }> }`. Every other port's runner reads this same file. + +- [ ] **Step 1: Write the corpus** + +```json +{ + "cases": [ + { + "name": "empty-scope-matches-everything", + "scope": {}, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "Order", "matches": true } + ] + }, + { + "name": "single-star-is-one-segment", + "scope": { "include": ["acme::*"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::commerce::Order", "matches": false }, + { "fqn": "other::Order", "matches": false } + ] + }, + { + "name": "double-star-is-one-or-more-segments", + "scope": { "include": ["acme::**"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::commerce::internal::Secret", "matches": true }, + { "fqn": "acme", "matches": false }, + { "fqn": "acmex::Order", "matches": false } + ] + }, + { + "name": "partial-star-never-crosses-separator", + "scope": { "include": ["acme::Order*"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::OrderLine", "matches": true }, + { "fqn": "acme::deep::OrderLine", "matches": false } + ] + }, + { + "name": "exclude-applied-after-include", + "scope": { "include": ["acme::**"], "exclude": ["acme::internal::**"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::internal::Secret", "matches": false } + ] + }, + { + "name": "exclude-alone-narrows-everything", + "scope": { "exclude": ["acme::internal::**"] }, + "expect": [ + { "fqn": "other::Thing", "matches": true }, + { "fqn": "acme::internal::Secret", "matches": false } + ] + }, + { + "name": "multiple-includes-are-a-union", + "scope": { "include": ["acme::commerce::**", "acme::common::**"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::common::BaseEntity", "matches": true }, + { "fqn": "acme::billing::Invoice", "matches": false } + ] + }, + { + "name": "regex-metacharacters-are-literal", + "scope": { "include": ["acme::Order.v2"] }, + "expect": [ + { "fqn": "acme::Order.v2", "matches": true }, + { "fqn": "acme::OrderXv2", "matches": false } + ] + } + ] +} +``` + +Write `fixtures/scope-conformance/README.md` stating: the corpus is the cross-port contract for `scope` pattern semantics; every port runs it; `*` matches any characters within one segment and never crosses `::`; a segment that is exactly `**` matches one or more segments; `include` empty means everything; `exclude` is applied after `include`. + +- [ ] **Step 2: Write the failing runner test** + +```ts +// server/typescript/packages/sdk/test/scope-conformance.test.ts +import { describe, test, expect } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { compileScope, matchesScope, type Scope } from "../src/scope.js"; + +interface Case { + name: string; + scope: Scope; + expect: Array<{ fqn: string; matches: boolean }>; +} + +const CORPUS = join(import.meta.dir, "../../../../../fixtures/scope-conformance/cases.json"); +const cases = (JSON.parse(readFileSync(CORPUS, "utf8")) as { cases: Case[] }).cases; + +describe("scope-conformance corpus", () => { + test("corpus is non-empty (a silent zero-case run is a failed gate)", () => { + expect(cases.length).toBeGreaterThan(0); + }); + for (const c of cases) { + test(c.name, () => { + const compiled = compileScope(c.scope); + for (const e of c.expect) { + expect({ fqn: e.fqn, matches: matchesScope(e.fqn, compiled) }) + .toEqual({ fqn: e.fqn, matches: e.matches }); + } + }); + } +}); +``` + +- [ ] **Step 3: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/scope-conformance.test.ts` +Expected: PASS — 9 tests (8 cases + the non-empty guard) + +- [ ] **Step 4: Prove the gate by breaking it** + +Temporarily change `SEGMENTS` in `scope.ts` to `".*"` (making `*` cross separators), re-run, and confirm `single-star-is-one-segment` FAILS. Then revert. + +Expected: FAIL before revert, PASS after. A gate that has never been seen red is not known to work. + +- [ ] **Step 5: Commit** + +```bash +git add fixtures/scope-conformance server/typescript/packages/sdk/test/scope-conformance.test.ts +git commit -m "test(conformance): scope-pattern corpus pins * and ** semantics cross-port" +``` + +--- + +## Task 3: Register new error codes + +**Files:** +- Modify: `server/typescript/packages/metadata/src/errors.ts:19` (the `ERROR_CODES` array) +- Test: `server/typescript/packages/metadata/test/errors.test.ts` (extend if present; create if not) + +**Interfaces:** +- Produces: `"ERR_SOURCE_UNRESOLVED"`, `"ERR_SOURCE_KIND_UNSUPPORTED"`, `"ERR_SCOPE_PATTERN_INVALID"`, `"ERR_COLLECTION_NOT_FOUND"` as members of `ERROR_CODES` + +- [ ] **Step 1: Write the failing test** + +```ts +import { describe, test, expect } from "bun:test"; +import { ERROR_CODES } from "../src/errors.js"; + +describe("phase-1 source-resolution error codes", () => { + test("are registered in the shared ledger", () => { + for (const code of [ + "ERR_SOURCE_UNRESOLVED", + "ERR_SOURCE_KIND_UNSUPPORTED", + "ERR_SCOPE_PATTERN_INVALID", + "ERR_COLLECTION_NOT_FOUND", + ]) { + expect(ERROR_CODES).toContain(code); + } + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/metadata/test/errors.test.ts` +Expected: FAIL — codes not found + +- [ ] **Step 3: Add the codes** + +Add these four string literals to the `ERROR_CODES` array in `errors.ts`, each with a comment naming the phase-1 source-resolution design as their origin, matching the file's existing comment style. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/metadata/test/errors.test.ts` +Expected: PASS + +- [ ] **Step 5: Note the cross-port debt** + +Add a line to the plan's tracking notes: Python `errors.py` (superset) and Java `ErrorCode.java` need the same four codes in the ports plan. TS `errors.ts` is exact-bidirectional, so its own gate will now expect them. + +- [ ] **Step 6: Commit** + +```bash +git add server/typescript/packages/metadata/src/errors.ts server/typescript/packages/metadata/test/errors.test.ts +git commit -m "feat(metadata): register phase-1 source-resolution error codes" +``` + +--- + +## Task 4: Source spec resolution + +**Files:** +- Create: `server/typescript/packages/sdk/src/sources.ts` +- Test: `server/typescript/packages/sdk/test/sources.test.ts` + +**Interfaces:** +- Produces: + - `type SourceSpec = { path: string } | { resource: string } | { package: string }` + - `interface ResolvedSource { readonly file: string; readonly spec: SourceSpec }` + - `function resolveSources(configDir: string, specs: readonly SourceSpec[]): Promise` — returns files sorted by absolute path (canonical, order-free); throws on an unresolvable `path`; throws `ERR_SOURCE_KIND_UNSUPPORTED` for `resource`/`package` in phase 1 + - `const DEFAULT_SOURCES: readonly SourceSpec[]` — `[{ path: "metaobjects" }]` + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/sdk/test/sources.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSources, DEFAULT_SOURCES } from "../src/sources.js"; + +let root: string; +const write = (rel: string, body = "{}") => { + const full = join(root, rel); + mkdirSync(join(full, ".."), { recursive: true }); + writeFileSync(full, body, "utf8"); + return full; +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-sources-")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("resolveSources", () => { + test("resolves a directory recursively, metadata files only", async () => { + write("model/meta.a.json"); + write("model/nested/meta.b.yaml"); + write("model/notes.txt"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out.map((r) => r.file.replace(root + "/", ""))).toEqual([ + "model/meta.a.json", + "model/nested/meta.b.yaml", + ]); + }); + + test("resolves a single file", async () => { + write("model/meta.a.json"); + const out = await resolveSources(root, [{ path: "model/meta.a.json" }]); + expect(out).toHaveLength(1); + }); + + test("output is canonically sorted regardless of spec order", async () => { + write("b/meta.b.json"); + write("a/meta.a.json"); + const forward = await resolveSources(root, [{ path: "a" }, { path: "b" }]); + const reverse = await resolveSources(root, [{ path: "b" }, { path: "a" }]); + expect(forward.map((r) => r.file)).toEqual(reverse.map((r) => r.file)); + }); + + test("de-duplicates a file contributed by two overlapping specs", async () => { + write("model/meta.a.json"); + const out = await resolveSources(root, [{ path: "model" }, { path: "model/meta.a.json" }]); + expect(out).toHaveLength(1); + }); + + test("paths resolve against the config dir, not process.cwd()", async () => { + write("apps/ui/.keep"); + write("model/meta.a.json"); + const out = await resolveSources(join(root, "apps/ui"), [{ path: "../../model" }]); + expect(out).toHaveLength(1); + }); + + test("an unresolvable path is ERR_SOURCE_UNRESOLVED, never a silent skip", async () => { + await expect(resolveSources(root, [{ path: "missing" }])).rejects.toThrow( + /ERR_SOURCE_UNRESOLVED/, + ); + }); + + test("resource and package kinds are ERR_SOURCE_KIND_UNSUPPORTED in phase 1", async () => { + await expect(resolveSources(root, [{ resource: "acme/model" }])).rejects.toThrow( + /ERR_SOURCE_KIND_UNSUPPORTED/, + ); + await expect(resolveSources(root, [{ package: "@acme/model" }])).rejects.toThrow( + /ERR_SOURCE_KIND_UNSUPPORTED/, + ); + }); + + test("_pending is excluded at any depth", async () => { + write("model/meta.a.json"); + write("model/_pending/meta.draft.json"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out).toHaveLength(1); + }); + + test("a nested symlinked directory is followed", async () => { + write("real/meta.b.json"); + write("model/meta.a.json"); + const { symlinkSync } = await import("node:fs"); + symlinkSync(join(root, "real"), join(root, "model/linked"), "dir"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out).toHaveLength(2); + }); + + test("DEFAULT_SOURCES is the metaobjects/ directory", () => { + expect(DEFAULT_SOURCES).toEqual([{ path: "metaobjects" }]); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/sources.test.ts` +Expected: FAIL — `Cannot find module '../src/sources.js'` + +- [ ] **Step 3: Write minimal implementation** + +```ts +// server/typescript/packages/sdk/src/sources.ts +import { readdir, stat } from "node:fs/promises"; +import { isAbsolute, join, resolve } from "node:path"; + +/** Tagged union of source kinds. `resource` and `package` are declared now so the + * config shape is stable; only `path` resolves in phase 1. */ +export type SourceSpec = + | { readonly path: string } + | { readonly resource: string } + | { readonly package: string }; + +export interface ResolvedSource { + /** Absolute path of one metadata file. */ + readonly file: string; + /** The spec that contributed it — provenance for diagnostics. */ + readonly spec: SourceSpec; +} + +/** Used when `sources` is absent or empty. `metaobjects/` is a DEFAULT, never a requirement. */ +export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: "metaobjects" }]; + +const PENDING_DIR = "_pending"; + +function isMetadataFile(name: string): boolean { + return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml"); +} + +/** Recursively collect metadata files. Uses `stat` (follows symlinks) so a symlinked + * subdirectory is traversed — the loader has always followed them. */ +async function collectDir(dir: string, out: string[]): Promise { + const entries = await readdir(dir); + for (const entry of entries) { + if (entry === PENDING_DIR) continue; + const full = join(dir, entry); + const s = await stat(full); + if (s.isDirectory()) await collectDir(full, out); + else if (s.isFile() && isMetadataFile(entry)) out.push(full); + } +} + +/** + * Resolve a source SET to a canonically-sorted list of metadata files. + * + * The result is sorted by absolute path and de-duplicated, so it is a pure function + * of the source set: permuting `specs` cannot change the output. Declared order + * carries no information (the loader derives whatever order it needs). + * + * @param configDir absolute directory of the declaring config — relative `path` + * specs resolve against it, never against ambient `process.cwd()`. + */ +export async function resolveSources( + configDir: string, + specs: readonly SourceSpec[], +): Promise { + const byFile = new Map(); + + for (const spec of specs) { + if (!("path" in spec)) { + const kind = "resource" in spec ? "resource" : "package"; + throw new Error( + `ERR_SOURCE_KIND_UNSUPPORTED: source kind "${kind}" is not supported by this ` + + `toolchain yet; use a "path" source.`, + ); + } + const target = isAbsolute(spec.path) ? spec.path : resolve(configDir, spec.path); + let s; + try { + s = await stat(target); + } catch { + throw new Error( + `ERR_SOURCE_UNRESOLVED: source path "${spec.path}" does not exist ` + + `(resolved to ${target}, relative to ${configDir}).`, + ); + } + const found: string[] = []; + if (s.isDirectory()) await collectDir(target, found); + else found.push(target); + for (const file of found) if (!byFile.has(file)) byFile.set(file, spec); + } + + return [...byFile.keys()] + .sort() + .map((file) => ({ file, spec: byFile.get(file)! })); +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/sources.test.ts` +Expected: PASS — 10 tests + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/sdk/src/sources.ts server/typescript/packages/sdk/test/sources.test.ts +git commit -m "feat(sdk): resolve a source SET to a canonically-sorted file list" +``` + +--- + +## Task 5: Config schema — `sources`, `scope`, `migrate.scope` + +**Files:** +- Modify: `server/typescript/packages/sdk/src/config.ts:64-78` +- Test: `server/typescript/packages/sdk/test/config.test.ts` + +**Interfaces:** +- Consumes: `SourceSpec` (Task 4), `Scope` (Task 1) +- Produces: `ConfigSchema` accepting `sources: SourceSpec[]`, `scope?: {include?: string[]; exclude?: string[]}`, and `migrate.scope?: string[]` + +- [ ] **Step 1: Write the failing test** + +Append to `packages/sdk/test/config.test.ts`: + +```ts +describe("ConfigSchema — phase-1 source resolution", () => { + test("accepts a path source", () => { + const p = ConfigSchema.parse({ schema_version: 1, sources: [{ path: "../model" }] }); + expect(p.sources).toEqual([{ path: "../model" }]); + }); + test("accepts resource and package source kinds", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + sources: [{ resource: "acme/model" }, { package: "@acme/model" }], + }); + expect(p.sources).toHaveLength(2); + }); + test("rejects an unknown source kind", () => { + expect(() => ConfigSchema.parse({ schema_version: 1, sources: [{ nope: "x" }] })).toThrow(); + }); + test("accepts a scope block", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + scope: { include: ["acme::**"], exclude: ["acme::internal::**"] }, + }); + expect(p.scope?.include).toEqual(["acme::**"]); + }); + test("scope defaults to undefined (match everything)", () => { + expect(ConfigSchema.parse({ schema_version: 1 }).scope).toBeUndefined(); + }); + test("accepts migrate.scope", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + migrate: { scope: ["acme::platform::**"] }, + }); + expect(p.migrate?.scope).toEqual(["acme::platform::**"]); + }); + test("an existing config with no new keys still parses (back-compat)", () => { + const p = ConfigSchema.parse({ + schema_version: 1, pending_in_git: true, + confidence_thresholds: { pending_promote: 0.8, drift_warn: 0.7 }, + sources: [], extract: {}, + }); + expect(p.sources).toEqual([]); + expect(p.scope).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/config.test.ts` +Expected: FAIL — the `resource` source and the `scope` block are rejected + +- [ ] **Step 3: Widen the schema** + +In `config.ts`, replace the existing `sources` union and add `scope`: + +```ts +const SourceSpecSchema = z.union([ + z.object({ path: z.string().min(1) }).strict(), + z.object({ resource: z.string().min(1) }).strict(), + z.object({ package: z.string().min(1) }).strict(), +]); + +const ScopeSchema = z.object({ + include: z.array(z.string().min(1)).optional(), + exclude: z.array(z.string().min(1)).optional(), +}).strict(); +``` + +In `ConfigSchema`: replace the `sources` field with `z.array(SourceSpecSchema).default([])`, add `scope: ScopeSchema.optional()`, and add `scope: z.array(z.string().min(1))` to `MigrateBlock`'s partial shape. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/config.test.ts` +Expected: PASS — including the pre-existing tests, unchanged + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/sdk/src/config.ts server/typescript/packages/sdk/test/config.test.ts +git commit -m "feat(sdk): config accepts a source SET, a scope block, and migrate.scope" +``` + +--- + +## Task 6: Nearest-ancestor discovery + +**Files:** +- Create: `server/typescript/packages/sdk/src/discovery.ts` +- Test: `server/typescript/packages/sdk/test/discovery.test.ts` + +**Interfaces:** +- Produces: `function findConfigDir(startDir: string): Promise` — walks up for a directory containing `.metaobjects/config.json`; stops after examining a directory containing `.git`; returns the containing directory (not the `.metaobjects` dir), or `undefined` + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/sdk/test/discovery.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { findConfigDir } from "../src/discovery.js"; + +let root: string; +const mk = (rel: string) => mkdirSync(join(root, rel), { recursive: true }); +const cfg = (rel: string) => { + mk(join(rel, ".metaobjects")); + writeFileSync(join(root, rel, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-discovery-")); mk(".git"); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("findConfigDir", () => { + test("finds a config in the start directory", async () => { + cfg("apps/ui"); mk("apps/ui/src"); + expect(await findConfigDir(join(root, "apps/ui"))).toBe(join(root, "apps/ui")); + }); + test("walks up to the nearest ancestor config", async () => { + cfg("apps/ui"); mk("apps/ui/src/deep"); + expect(await findConfigDir(join(root, "apps/ui/src/deep"))).toBe(join(root, "apps/ui")); + }); + test("nearest wins over a further ancestor", async () => { + cfg("."); cfg("apps/ui"); mk("apps/ui/src"); + expect(await findConfigDir(join(root, "apps/ui/src"))).toBe(join(root, "apps/ui")); + }); + test("stops at the repository boundary — never adopts a parent checkout's config", async () => { + // A config ABOVE the .git boundary must not be found. + const outer = mkdtempSync(join(tmpdir(), "metaobjects-outer-")); + try { + mkdirSync(join(outer, "inner/.git"), { recursive: true }); + mkdirSync(join(outer, ".metaobjects"), { recursive: true }); + writeFileSync(join(outer, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); + mkdirSync(join(outer, "inner/src"), { recursive: true }); + expect(await findConfigDir(join(outer, "inner/src"))).toBeUndefined(); + } finally { + rmSync(outer, { recursive: true, force: true }); + } + }); + test("a repo-root config IS found from a subdirectory", async () => { + cfg("."); mk("apps/ui"); + expect(await findConfigDir(join(root, "apps/ui"))).toBe(root); + }); + test("returns undefined when nothing is found", async () => { + mk("apps/ui"); + expect(await findConfigDir(join(root, "apps/ui"))).toBeUndefined(); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/discovery.test.ts` +Expected: FAIL — `Cannot find module '../src/discovery.js'` + +- [ ] **Step 3: Write minimal implementation** + +```ts +// server/typescript/packages/sdk/src/discovery.ts +import { stat } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { DEFAULT_METAOBJECTS_DIR } from "./memory.js"; + +const CONFIG_FILE = "config.json"; +const GIT_DIR = ".git"; + +async function exists(p: string): Promise { + try { await stat(p); return true; } catch { return false; } +} + +/** + * Walk up from `startDir` for the nearest directory holding + * `.metaobjects/config.json`. The walk STOPS after examining a directory that + * contains `.git`, so a monorepo can never silently adopt a parent checkout's + * configuration. Returns the containing directory, or undefined. + */ +export async function findConfigDir(startDir: string): Promise { + let dir = resolve(startDir); + for (;;) { + if (await exists(join(dir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE))) return dir; + // Boundary check AFTER the config check: a repo-root config is still findable. + if (await exists(join(dir, GIT_DIR))) return undefined; + const parent = dirname(dir); + if (parent === dir) return undefined; + dir = parent; + } +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/discovery.test.ts` +Expected: PASS — 6 tests + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/sdk/src/discovery.ts server/typescript/packages/sdk/test/discovery.test.ts +git commit -m "feat(sdk): nearest-ancestor config discovery, bounded by the repo root" +``` + +--- + +## Task 7: `resolveCollection()` — the single authority + +**Files:** +- Create: `server/typescript/packages/sdk/src/collection.ts` +- Modify: `server/typescript/packages/sdk/src/index.ts` (export the new surface) +- Test: `server/typescript/packages/sdk/test/collection.test.ts` + +**Interfaces:** +- Consumes: `findConfigDir` (T6), `resolveSources`/`DEFAULT_SOURCES` (T4), `compileScope` (T1), `loadConfig` (T5) +- Produces: + - `interface Collection { readonly configDir: string; readonly files: readonly string[]; readonly sources: readonly ResolvedSource[]; readonly scope: CompiledScope; readonly migrateScope: CompiledScope | undefined }` + - `function resolveCollection(startDir: string, opts?: { explicitDir?: string }): Promise` — throws `ERR_COLLECTION_NOT_FOUND` when no config is discovered AND the default directory does not exist + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/sdk/test/collection.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveCollection } from "../src/collection.js"; +import { matchesScope } from "../src/scope.js"; + +let root: string; +const write = (rel: string, body: string) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body, "utf8"); +}; +const config = (dir: string, cfg: object) => + write(join(dir, ".metaobjects/config.json"), JSON.stringify({ schema_version: 1, ...cfg })); + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-collection-")); mkdirSync(join(root, ".git")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("resolveCollection", () => { + test("BACK-COMPAT: no sources declared falls back to metaobjects/", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + const c = await resolveCollection(root); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["metaobjects/meta.a.json"]); + }); + + test("BACK-COMPAT: no config at all still finds metaobjects/ in the start dir", async () => { + write("metaobjects/meta.a.json", "{}"); + const c = await resolveCollection(root); + expect(c.files).toHaveLength(1); + }); + + test("a consumer reaches a tree elsewhere in the repo", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui")); + expect(c.configDir).toBe(join(root, "apps/ui")); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); + }); + + test("scope compiles and is applied by matchesScope", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }], scope: { include: ["acme::**"] } }); + const c = await resolveCollection(join(root, "apps/ui")); + expect(matchesScope("acme::Order", c.scope)).toBe(true); + expect(matchesScope("other::Order", c.scope)).toBe(false); + }); + + test("migrateScope is undefined when not declared", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + expect((await resolveCollection(root)).migrateScope).toBeUndefined(); + }); + + test("migrateScope compiles when declared", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", { migrate: { scope: ["acme::platform::**"] } }); + const c = await resolveCollection(root); + expect(matchesScope("acme::platform::Job", c.migrateScope!)).toBe(true); + expect(matchesScope("arena::Match", c.migrateScope!)).toBe(false); + }); + + test("an explicit dir overrides discovery", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + config("apps/api", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui"), { explicitDir: join(root, "apps/api") }); + expect(c.configDir).toBe(join(root, "apps/api")); + }); + + test("nothing discoverable and no default dir is ERR_COLLECTION_NOT_FOUND", async () => { + mkdirSync(join(root, "apps/ui"), { recursive: true }); + await expect(resolveCollection(join(root, "apps/ui"))).rejects.toThrow( + /ERR_COLLECTION_NOT_FOUND/, + ); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/collection.test.ts` +Expected: FAIL — `Cannot find module '../src/collection.js'` + +- [ ] **Step 3: Write minimal implementation** + +```ts +// server/typescript/packages/sdk/src/collection.ts +import { stat } from "node:fs/promises"; +import { join, resolve } from "node:path"; +import { loadConfig } from "./config.js"; +import { findConfigDir } from "./discovery.js"; +import { compileScope, type CompiledScope } from "./scope.js"; +import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./memory.js"; +import { DEFAULT_SOURCES, resolveSources, type ResolvedSource, type SourceSpec } from "./sources.js"; + +export interface Collection { + /** Directory whose config declared this collection. */ + readonly configDir: string; + /** Canonically-sorted absolute metadata file paths. */ + readonly files: readonly string[]; + /** Same set, carrying the contributing spec for provenance. */ + readonly sources: readonly ResolvedSource[]; + /** Output filter for codegen. Empty include => everything. */ + readonly scope: CompiledScope; + /** Output filter for migrate/verify --db. Undefined => the command governs everything in scope. */ + readonly migrateScope: CompiledScope | undefined; +} + +async function isDir(p: string): Promise { + try { return (await stat(p)).isDirectory(); } catch { return false; } +} + +/** + * THE single authority on where metadata lives. Every read path routes through + * this — `metaobjects/` is the DEFAULT value of `sources`, never an assumption + * baked into a call site. + */ +export async function resolveCollection( + startDir: string, + opts?: { explicitDir?: string }, +): Promise { + const explicit = opts?.explicitDir; + const configDir = explicit !== undefined + ? resolve(explicit) + : (await findConfigDir(startDir)) ?? resolve(startDir); + + let specs: readonly SourceSpec[] = DEFAULT_SOURCES; + let scopeSpec = undefined as { include?: string[]; exclude?: string[] } | undefined; + let migrateSpec: string[] | undefined; + + if (await isDir(join(configDir, DEFAULT_METAOBJECTS_DIR))) { + try { + const cfg = await loadConfig(join(configDir, DEFAULT_METAOBJECTS_DIR)); + if (cfg.sources.length > 0) specs = cfg.sources; + scopeSpec = cfg.scope; + migrateSpec = cfg.migrate?.scope; + } catch { + // No config.json, or unreadable — fall through to the default source set. + // A malformed config surfaces from loadConfig on the paths that require it. + } + } + + // Only the DEFAULT is allowed to be absent — an explicitly declared source that + // does not resolve is an error (resolveSources throws ERR_SOURCE_UNRESOLVED). + if (specs === DEFAULT_SOURCES && !(await isDir(join(configDir, DEFAULT_METADATA_DIR)))) { + throw new Error( + `ERR_COLLECTION_NOT_FOUND: no metadata sources declared in ${configDir} and no ` + + `default "${DEFAULT_METADATA_DIR}" directory found. Declare "sources" in ` + + `${DEFAULT_METAOBJECTS_DIR}/config.json, or run 'meta init' to scaffold.`, + ); + } + + const sources = await resolveSources(configDir, specs); + return { + configDir, + files: sources.map((s) => s.file), + sources, + scope: compileScope(scopeSpec ?? {}), + migrateScope: migrateSpec === undefined ? undefined : compileScope({ include: migrateSpec }), + }; +} +``` + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/collection.test.ts` +Expected: PASS — 8 tests + +- [ ] **Step 5: Export the surface** + +Add to `packages/sdk/src/index.ts`: `resolveCollection`, `type Collection` from `./collection.js`; `compileScope`, `matchesScope`, `type Scope`, `type CompiledScope` from `./scope.js`; `resolveSources`, `DEFAULT_SOURCES`, `type SourceSpec`, `type ResolvedSource` from `./sources.js`; `findConfigDir` from `./discovery.js`. + +- [ ] **Step 6: Typecheck and commit** + +```bash +cd server/typescript && bun run --filter '@metaobjectsdev/sdk' typecheck +git add server/typescript/packages/sdk/src/collection.ts server/typescript/packages/sdk/src/index.ts server/typescript/packages/sdk/test/collection.test.ts +git commit -m "feat(sdk): resolveCollection() — one authority for where metadata lives" +``` + +--- + +## Task 8: Order-independence gate (the linchpin) + +Without this, set semantics is a belief that decays the first time someone writes an order-sensitive code path. + +**Files:** +- Test: `server/typescript/packages/sdk/test/order-independence.test.ts` + +**Interfaces:** +- Consumes: `resolveSources` (T4), `loadMemory` (existing), the canonical serializer from `@metaobjectsdev/metadata` + +- [ ] **Step 1: Write the test** + +```ts +// server/typescript/packages/sdk/test/order-independence.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSources, type SourceSpec } from "../src/sources.js"; + +let root: string; +const write = (rel: string, body: object) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), JSON.stringify(body), "utf8"); +}; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "metaobjects-order-")); + // A base declaration, an overlay onto it, and an independent third file — + // the shapes whose merge is order-sensitive if anything is. + write("a/meta.base.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }] }, + }); + write("b/meta.overlay.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", overlay: true, children: [ + { "field.string": { name: "note" } }] } }] }, + }); + write("c/meta.other.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Customer", children: [{ "field.string": { name: "id" } }] } }] }, + }); +}); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +function permutations(items: T[]): T[][] { + if (items.length <= 1) return [items]; + const out: T[][] = []; + for (let i = 0; i < items.length; i++) { + const rest = [...items.slice(0, i), ...items.slice(i + 1)]; + for (const p of permutations(rest)) out.push([items[i]!, ...p]); + } + return out; +} + +describe("order independence", () => { + test("resolveSources output is identical across every spec permutation", async () => { + const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; + const results = await Promise.all( + permutations(specs).map((p) => resolveSources(root, p).then((r) => r.map((x) => x.file))), + ); + expect(results).toHaveLength(6); + for (const r of results) expect(r).toEqual(results[0]!); + }); + + test("the loaded model serializes byte-identically across every permutation", async () => { + const { MetaDataLoader, composeRegistry, coreProviders, serializeCanonical } = + await import("@metaobjectsdev/metadata"); + const { FileSource } = await import("@metaobjectsdev/metadata/core"); + const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; + + const serialized: string[] = []; + for (const p of permutations(specs)) { + const resolved = await resolveSources(root, p); + const loader = new MetaDataLoader({ registry: composeRegistry(coreProviders) }); + const result = await loader.load(resolved.map((r) => new FileSource(r.file))); + expect(result.errors).toHaveLength(0); + serialized.push(serializeCanonical(result.root)); + } + expect(serialized).toHaveLength(6); + for (const s of serialized) expect(s).toBe(serialized[0]!); + }); +}); +``` + +- [ ] **Step 2: Run it** + +Run: `cd server/typescript && bun test packages/sdk/test/order-independence.test.ts` +Expected: PASS. + +**If the second test FAILS, stop and report before changing anything.** A failure means the loader is not in fact order-independent for this shape, which invalidates a load-bearing premise of the design — that is a finding to escalate, not a test to adjust. + +**Note for the implementer:** confirm the exact export name of the canonical serializer (`serializeCanonical` above is the expected name) by grepping `packages/metadata/src/index.ts`; use the real export and adjust the import. + +- [ ] **Step 3: Prove the gate by breaking it** + +Temporarily remove the `.sort()` from `resolveSources` in `sources.ts`, re-run, and confirm the first test FAILS. Revert. + +Expected: FAIL before revert, PASS after. + +- [ ] **Step 4: Commit** + +```bash +git add server/typescript/packages/sdk/test/order-independence.test.ts +git commit -m "test(sdk): pin order independence — permuted source sets serialize byte-identically" +``` + +--- + +## Task 9: Route `loadMemory` through a resolved collection + +**Files:** +- Modify: `server/typescript/packages/sdk/src/memory.ts:105,122-143` +- Test: `server/typescript/packages/sdk/test/memory.test.ts` + +**Interfaces:** +- Consumes: `Collection` (T7) +- Produces: `loadMemory(repoRoot: string, options?: LoadMemoryOptions & { files?: readonly string[] })` — when `files` is supplied it loads exactly those and skips all directory discovery; behavior with `files` absent is unchanged + +- [ ] **Step 1: Write the failing test** + +Append to `packages/sdk/test/memory.test.ts`: + +```ts +describe("loadMemory with an explicit file set", () => { + test("loads exactly the supplied files, ignoring any metaobjects/ dir", async () => { + const dir = mkdtempSync(join(tmpdir(), "metaobjects-memory-files-")); + try { + mkdirSync(join(dir, "model"), { recursive: true }); + mkdirSync(join(dir, "metaobjects"), { recursive: true }); + writeFileSync(join(dir, "model/meta.a.json"), JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }] }, + }), "utf8"); + writeFileSync(join(dir, "metaobjects/meta.decoy.json"), JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Decoy", children: [{ "field.string": { name: "id" } }] } }] }, + }), "utf8"); + const root = await loadMemory(dir, { files: [join(dir, "model/meta.a.json")] }); + const names = root.children().map((c) => c.name); + expect(names).toContain("Order"); + expect(names).not.toContain("Decoy"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/sdk/test/memory.test.ts` +Expected: FAIL — `Decoy` is present, because `loadMemory` still scans `metaobjects/` + +- [ ] **Step 3: Implement** + +In `memory.ts`, add `files?: readonly string[]` to `LoadMemoryOptions`, and in `loadMemory` replace the `collectMetadataPaths(repoRoot)` call with: + +```ts +const paths = options?.files !== undefined + ? [...options.files] + : await collectMetadataPaths(repoRoot); +``` + +Leave `collectMetadataPaths` and `listMetadataFiles` untouched — they remain the no-`files` fallback and the back-compat path. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/sdk/test/memory.test.ts` +Expected: PASS — including all pre-existing tests + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/sdk/src/memory.ts server/typescript/packages/sdk/test/memory.test.ts +git commit -m "feat(sdk): loadMemory accepts an explicit resolved file set" +``` + +--- + +## Task 10: Route the CLI read sites + +Five of the nine hardcoded reads. `init.ts` is deliberately excluded — it writes the default. + +**Files:** +- Modify: `server/typescript/packages/cli/src/commands/gen.ts:55-72` +- Modify: `server/typescript/packages/cli/src/commands/docs.ts:292,529-530` +- Modify: `server/typescript/packages/cli/src/commands/export.ts:19` +- Modify: `server/typescript/packages/cli/src/index.ts:275` +- Test: `server/typescript/packages/cli/test/collection-routing.test.ts` (create) + +**Interfaces:** +- Consumes: `resolveCollection` (T7), `loadMemory({ files })` (T9) + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/cli/test/collection-routing.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { genCommand } from "../src/commands/gen.js"; + +let root: string; +const write = (rel: string, body: string) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body, "utf8"); +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-cli-route-")); mkdirSync(join(root, ".git")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("gen routes metadata discovery through resolveCollection", () => { + test("generates from a sources-declared tree with no metaobjects/ present", async () => { + write("model/meta.a.json", JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [ + { "field.string": { name: "id" } }, + { "identity.primary": { "@fields": ["id"] } }, + { "source.rdb": { "@table": "orders", "@kind": "table" } }] } }] }, + })); + write("apps/ui/.metaobjects/config.json", JSON.stringify({ + schema_version: 1, sources: [{ path: "../../model" }], + })); + write("apps/ui/metaobjects.config.ts", [ + 'import { defineConfig } from "@metaobjectsdev/cli";', + 'import { entityFile } from "@metaobjectsdev/codegen-ts/generators";', + 'export default defineConfig({ outDir: "./src/generated", dialect: "postgres",', + ' dbImport: "../db", generators: [entityFile()] });', + ].join("\n")); + + const code = await genCommand({ cwd: join(root, "apps/ui") } as never); + expect(code).toBe(0); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/cli/test/collection-routing.test.ts` +Expected: FAIL — exit code 2, `no metaobjects/ found` + +**Note for the implementer:** `genCommand`'s real parameter shape is in `packages/cli/src/commands/gen.ts`; adjust the call to match it rather than the `as never` placeholder above. + +- [ ] **Step 3: Route gen.ts** + +Replace the `loadMemory(projectRoot, …)` call and its error branch: + +```ts +let collection; +try { + collection = await resolveCollection(projectRoot); +} catch (err) { + log.error((err as Error).message); + return 2; +} + +let metadata; +try { + metadata = await loadMemory(collection.configDir, { + files: collection.files, + ...(forgeConfig.providers !== undefined ? { providers: forgeConfig.providers } : {}), + }); +} catch (err) { + log.error(`failed to load metadata: ${(err as Error).message}`); + return 2; +} +``` + +The `existsSync(join(projectRoot, DEFAULT_METADATA_DIR))` hint branch is deleted — `resolveCollection` now raises `ERR_COLLECTION_NOT_FOUND` with a better message, and the comment above that branch (about not swallowing genuine ParseErrors) is satisfied by construction since the two failure modes are now separate `try` blocks. + +- [ ] **Step 4: Route the remaining four sites** + +- `export.ts:19` — replace `join(projectRoot, DEFAULT_METADATA_DIR)` with `(await resolveCollection(projectRoot)).files`, passing them to the loader. +- `docs.ts:292` — replace the `existsSync` guard with a `resolveCollection` call inside a `try`, reporting its error message. +- `docs.ts:529-530` — `sourceDirs` becomes the collection's `configDir`-relative source dirs; derive `seenBasenames` from the resolved sources rather than the literal. +- `index.ts:275` — the "is this a MetaObjects project?" probe becomes `await resolveCollection(cwd).then(() => true).catch(() => false)`. + +- [ ] **Step 5: Run the full CLI suite for back-compat** + +Run: `cd server/typescript && bun test packages/cli` +Expected: PASS — every pre-existing test unchanged. A project with `metaobjects/` at the root and no `sources` must behave exactly as before. + +- [ ] **Step 6: Run the golden-output gate** + +Run: `cd server/typescript && bun test packages/codegen-ts` +Expected: PASS — generated output byte-identical. (`codegen-ts/test/golden/` lives outside the package under change and is the gate that catches accidental output drift.) + +- [ ] **Step 7: Commit** + +```bash +git add server/typescript/packages/cli/src server/typescript/packages/cli/test/collection-routing.test.ts +git commit -m "feat(cli): route gen/docs/export and the project probe through resolveCollection" +``` + +--- + +## Task 11: `detect-stack` routing and the nested-symlink fix + +Closes the divergence where `detect-stack` and the loader disagree about the same tree. + +**Files:** +- Modify: `server/typescript/packages/cli/src/lib/detect-stack.ts:24,31-69` +- Test: `server/typescript/packages/cli/test/detect-stack.test.ts` (extend if present; create if not) + +**Interfaces:** +- Consumes: `resolveCollection` (T7) +- Produces: `resolveStack(cwd, overrides)` unchanged in signature; `hasRequirementNodes` now scans the resolved collection's files + +- [ ] **Step 1: Write the failing tests** + +```ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveStack } from "../src/lib/detect-stack.js"; + +let root: string; +const write = (rel: string, body: string) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body, "utf8"); +}; +const REQ = JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "requirement.functional": { name: "FR1", "@level": 1, "@status": "live" } }] }, +}); + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-detect-")); mkdirSync(join(root, ".git")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("detect-stack honours sources", () => { + test("finds requirement nodes in a sources-declared tree", async () => { + write("model/meta.req.json", REQ); + write("apps/ui/.metaobjects/config.json", JSON.stringify({ + schema_version: 1, sources: [{ path: "../../model" }], + })); + const stack = await resolveStack(join(root, "apps/ui"), { servers: [], clients: [] }); + expect(stack.concerns).toContain("requirements"); + }); + + test("finds requirement nodes behind a NESTED symlinked directory", async () => { + write("real/meta.req.json", REQ); + write("metaobjects/meta.a.json", "{}"); + symlinkSync(join(root, "real"), join(root, "metaobjects/linked"), "dir"); + const stack = await resolveStack(root, { servers: [], clients: [] }); + expect(stack.concerns).toContain("requirements"); + }); +}); +``` + +- [ ] **Step 2: Run to verify they fail** + +Run: `cd server/typescript && bun test packages/cli/test/detect-stack.test.ts` +Expected: FAIL — both. The first because `sources` is ignored; the second because `Dirent.isDirectory()` is `false` for a symlinked directory, so the walk never descends. + +- [ ] **Step 3: Implement** + +Make `resolveStack` and `probe` async. Replace `hasRequirementNodes(cwd)` with a scan over `(await resolveCollection(cwd)).files` — reading each file and testing for the `REQUIREMENT_NODE_MARKER` substring — wrapped in a `try`/`catch` that returns `false`, preserving the existing "this is a cheap heuristic, never throws" contract. Delete the `METADATA_DIR` constant and the bespoke `readdirSync` walk entirely; the symlink bug disappears with the walk, because `resolveSources` uses `stat` (which follows). + +Update `resolveStack`'s callers to `await` it. + +- [ ] **Step 4: Run to verify they pass** + +Run: `cd server/typescript && bun test packages/cli` +Expected: PASS — both new tests, and every pre-existing detect-stack and agent-context test + +- [ ] **Step 5: Commit** + +```bash +git add server/typescript/packages/cli/src/lib/detect-stack.ts server/typescript/packages/cli/test/detect-stack.test.ts +git commit -m "fix(cli): detect-stack reads the resolved collection, fixing nested-symlink blindness" +``` + +--- + +## Task 12: Per-command scope for `migrate` and `verify --db` + +Without this, load-everything converts a real adopter's worst hazard — a `--from-db` migrate proposing to drop tables it does not model — from a discipline into an automation. + +**Files:** +- Modify: `server/typescript/packages/cli/src/commands/migrate.ts:260` and the expected-schema construction +- Test: `server/typescript/packages/cli/test/migrate-scope.test.ts` (create) + +**Interfaces:** +- Consumes: `Collection.migrateScope` (T7), `matchesScope` (T1) + +- [ ] **Step 1: Write the failing test** + +```ts +// server/typescript/packages/cli/test/migrate-scope.test.ts +import { describe, test, expect } from "bun:test"; +import { compileScope, matchesScope } from "@metaobjectsdev/sdk"; +import { scopeExpectedSchema } from "../src/commands/migrate.js"; + +describe("migrate scope", () => { + test("objects outside migrateScope are excluded from the expected schema", () => { + const expected = { + tables: [ + { name: "jobs", fqn: "acme::platform::Job" }, + { name: "matches", fqn: "arena::Match" }, + ], + views: [], + }; + const scoped = scopeExpectedSchema(expected as never, compileScope({ include: ["acme::platform::**"] })); + expect(scoped.tables.map((t) => t.name)).toEqual(["jobs"]); + }); + + test("an undefined scope leaves the expected schema untouched", () => { + const expected = { tables: [{ name: "jobs", fqn: "acme::platform::Job" }], views: [] }; + expect(scopeExpectedSchema(expected as never, undefined)).toEqual(expected as never); + }); + + test("matchesScope drives the decision (no second pattern implementation)", () => { + const c = compileScope({ include: ["acme::platform::**"] }); + expect(matchesScope("acme::platform::Job", c)).toBe(true); + expect(matchesScope("arena::Match", c)).toBe(false); + }); +}); +``` + +- [ ] **Step 2: Run to verify it fails** + +Run: `cd server/typescript && bun test packages/cli/test/migrate-scope.test.ts` +Expected: FAIL — `scopeExpectedSchema` is not exported + +- [ ] **Step 3: Implement** + +Export from `migrate.ts`: + +```ts +/** + * Narrow an expected schema to the objects inside `scope`. Tables and views whose + * declaring object falls outside are dropped BEFORE the diff, so the migration + * neither creates nor drops them — they belong to another owner. + */ +export function scopeExpectedSchema( + expected: ExpectedSchema, + scope: CompiledScope | undefined, +): ExpectedSchema { + if (scope === undefined) return expected; + return { + ...expected, + tables: expected.tables.filter((t) => matchesScope(t.fqn, scope)), + views: expected.views.filter((v) => matchesScope(v.fqn, scope)), + }; +} +``` + +Call it on the expected schema immediately before `diff()`, passing `collection.migrateScope`. + +**Note for the implementer:** confirm the real `ExpectedSchema` shape and whether its table/view entries already carry the declaring object's FQN. If they do not, thread it through at construction — do **not** re-derive an FQN from the SQL name, which is lossy. + +- [ ] **Step 4: Run to verify it passes** + +Run: `cd server/typescript && bun test packages/cli/test/migrate-scope.test.ts` +Expected: PASS + +- [ ] **Step 5: Run the migrate suites** + +Run: `cd server/typescript && bun test packages/migrate-ts && bun test packages/cli/test` +Expected: PASS. **A project with no `migrate.scope` must emit byte-identical migrations** — that is the back-compat guarantee for this task. + +- [ ] **Step 6: Commit** + +```bash +git add server/typescript/packages/cli/src/commands/migrate.ts server/typescript/packages/cli/test/migrate-scope.test.ts +git commit -m "feat(cli): migrate.scope narrows the expected schema so unowned tables are never touched" +``` + +--- + +## Task 13: Dogfood against the in-repo examples tree + +Proves reach and scope against a real metadata tree with zero new content. + +**Files:** +- Test: `server/typescript/packages/sdk/test/dogfood-examples.test.ts` (create) + +**Interfaces:** +- Consumes: `resolveCollection` (T7), `matchesScope` (T1) + +- [ ] **Step 1: Inspect the tree and read its declared package** + +Run: `ls examples/advanced-modeling/metaobjects && head -5 examples/advanced-modeling/metaobjects/meta.catalog.yaml` + +Record the actual `package:` value — the test below must assert against the real package, not a guess. + +- [ ] **Step 2: Write the test** + +```ts +// server/typescript/packages/sdk/test/dogfood-examples.test.ts +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { resolveCollection } from "../src/collection.js"; +import { loadMemory } from "../src/memory.js"; + +const EXAMPLES = resolve(import.meta.dir, "../../../../../examples/advanced-modeling/metaobjects"); + +let consumer: string; +beforeEach(() => { + consumer = mkdtempSync(join(tmpdir(), "metaobjects-dogfood-")); + mkdirSync(join(consumer, ".git")); + mkdirSync(join(consumer, "apps/ui/.metaobjects"), { recursive: true }); + writeFileSync( + join(consumer, "apps/ui/.metaobjects/config.json"), + JSON.stringify({ schema_version: 1, sources: [{ path: EXAMPLES }] }), + "utf8", + ); +}); +afterEach(() => { rmSync(consumer, { recursive: true, force: true }); }); + +describe("dogfood: a consumer reaches the in-repo examples tree", () => { + test("resolves every metadata file in it", async () => { + const c = await resolveCollection(join(consumer, "apps/ui")); + expect(c.files.length).toBeGreaterThanOrEqual(3); + expect(c.files.every((f) => f.startsWith(EXAMPLES))).toBe(true); + }); + + test("the resolved set loads without errors", async () => { + const c = await resolveCollection(join(consumer, "apps/ui")); + const root = await loadMemory(c.configDir, { files: c.files }); + expect(root.children().length).toBeGreaterThan(0); + }); +}); +``` + +- [ ] **Step 3: Run it** + +Run: `cd server/typescript && bun test packages/sdk/test/dogfood-examples.test.ts` +Expected: PASS + +- [ ] **Step 4: Commit** + +```bash +git add server/typescript/packages/sdk/test/dogfood-examples.test.ts +git commit -m "test(sdk): dogfood reach+scope against the in-repo examples metadata tree" +``` + +--- + +## Task 14: Documentation + +**Files:** +- Modify: `server/typescript/packages/cli/README.md` +- Modify: `CLAUDE.md` (the "File organization" and "Other conventions" sections) +- Create: `docs/features/metadata-sources.md` + +- [ ] **Step 1: Write the adopter guide** + +`docs/features/metadata-sources.md` covering: `sources` as a set with `metaobjects/` as its default; the `path` source kind (relative to the declaring config, read in place, never installed); `scope` with `*`/`**` semantics and include/exclude; nearest-ancestor discovery and the `.git` boundary; `migrate.scope` and the rule that a `migrate` block belongs where the ledger lives; and a **vendoring** section stating that airgapped builds are served by copying a dependency into a directory and pointing a `path` at it — no separate mechanism needed. + +Include one worked polyglot example (generic names only — no real project names). + +- [ ] **Step 2: Update the CLI README** + +Document `sources`, `scope`, and `migrate.scope` in the config reference, next to the existing `targets` documentation. + +- [ ] **Step 3: Update CLAUDE.md** + +In "File organization", state that `metaobjects/` is the **default** value of `sources` and never a requirement. In "Other conventions", add one line: metadata location is resolved via `resolveCollection()`; no code path may hardcode the directory name except `meta init`, which scaffolds it. + +- [ ] **Step 4: Leak scan and commit** + +```bash +grep -rniE "party|/home/" docs/features/metadata-sources.md && echo LEAK || echo clean +git add docs/features/metadata-sources.md server/typescript/packages/cli/README.md CLAUDE.md +git commit -m "docs: metadata sources, scope, discovery, and the vendoring workflow" +``` + +--- + +## Task 15: Full-suite verification + +- [ ] **Step 1: Build the workspace** + +Run, from the repository root: `bun run --filter '*' build` +Expected: success + +- [ ] **Step 2: Typecheck the workspace** + +Run: `bun run --filter '*' typecheck` +Expected: no errors. (`bun test` transpiles per-file and does not typecheck, so this is the gate that catches type breakage.) + +- [ ] **Step 3: Run the server suite** + +Run: `cd server/typescript && bun test` +Expected: PASS + +- [ ] **Step 4: Run the client suites** + +Run each `client/web/packages/` suite. +Expected: PASS + +- [ ] **Step 5: Confirm no hardcoded reads remain** + +Run: `git grep -n "DEFAULT_METADATA_DIR" -- 'server/typescript/packages/cli/src/**' 'server/typescript/packages/sdk/src/**'` +Expected: hits only in `memory.ts` (the constant's definition plus the no-`files` fallback), `sources.ts` (`DEFAULT_SOURCES`), `collection.ts` (the default check), and `init.ts` (scaffolding). **Any hit in `docs.ts`, `export.ts`, `gen.ts`, `index.ts`, or `detect-stack.ts` is an unfinished task.** + +- [ ] **Step 6: Commit any fixes and push** + +```bash +git add -- # never `git add -A` +git commit -m "chore: phase-1 source resolution full-suite verification" +``` + +--- + +## Self-Review Notes + +**Spec coverage.** §4.1 set semantics → T4, T8. §4.2 source kinds → T4, T5. §4.3 scope at output → T1, T2. §4.4 scope attachment incl. per-command → T12. §4.5 precedence → *not implemented in phase 1*: local-vs-dependency precedence only becomes reachable once `package` sources exist, and phase 1 rejects them (T4). §4.6/4.6.0 one authority → T7, T10, T11. §4.6.1 discovery → T6. §4.6.2 schema ownership → T12 (the `migrate.scope` half; the "ledger marks the owner" rule is documentation, T14). §4.7 conformance → T2, T8. §4.9 naming → T5. §8 symlink divergence → T11. §8 dogfood → T13. + +**Deliberately deferred to the ports plan:** C#, Java, Kotlin and Python implementations; the Python `metadata:`-string → set widening; Java's `scope` element alongside legacy ``; the four error codes in `errors.py` and `ErrorCode.java`; port runners for the scope-conformance corpus. + +**Deliberately out of scope:** the first-party shared metadata collection (separate deliverable); named `collection` references (§6); `url` sources; everything in §10 (issues #299–#306). + +**Three places the implementer must verify against real code rather than trusting this plan:** the canonical serializer's export name (T8 Step 2), `genCommand`'s parameter shape (T10 Step 2), and whether `ExpectedSchema` entries carry a declaring FQN (T12 Step 3). Each is flagged inline. diff --git a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md new file mode 100644 index 000000000..4a8725883 --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-design.md @@ -0,0 +1,574 @@ +# Metadata source resolution — collections, scope, and discovery (design) + +_Status: PROPOSED (awaiting review; nothing implemented)._ +_Date: 2026-08-17._ +_Prior art: `2026-08-17-metadata-source-resolution-prior-art.md`._ +_Supersedes the scope of `2026-06-11-fr-023-metadata-packages-design.md`, which becomes one +resolver inside this design rather than the whole feature._ + +## 1. Problem + +Four of five ports resolve metadata as exactly one directory per invocation. The Java port is the +exception: its Maven loader config takes an ordered list of sources, each expressible as a file, +URL, or classpath resource. Nothing in the standard ever absorbed that, and nothing in any port +lets a consumer say "the model lives over there, and I want this part of it." + +Two adopter shapes are blocked on it today, and both are polyglot monorepos. + +**A polyglot Maven + TypeScript repo.** One authored metadata tree lives in a dedicated Maven +module's `src/main/resources/metadata/`. Four Java modules consume it, each through a +hand-enumerated list of individual files in its `pom.xml` — 91, 63, 1 and 3 entries respectively. +Two TypeScript apps consume the same tree and cannot enumerate anything, because the Node CLI only +ever looks at `/metaobjects`; they reach it by **directory symlink**. Four metadata files +exist on disk that the largest list omits, and nothing distinguishes an intentional omission from +a forgotten one. + +**A two-rail repo over one database.** A Java rail and a TypeScript rail, two metadata homes, one +Postgres. Its own architecture memo records the blocker precisely: *there is no cross-project +include mechanism, so two directories sharing a common package requires symlinks or copy-sync — a +new drift class.* The chosen workaround is to consolidate into one home; the price paid in the +meantime is entities modeled twice with hand-maintained "keep in lockstep" comments, and a staging +script that copies a subset into a temporary directory to fake what the toolchain will not do. + +A third adopter — a single-app TypeScript repo on the current release — is the control. It needs +nothing, and must keep needing nothing. + +## 2. The reframing + +The obvious reading is that these repos need to *compose* several metadata collections. They do +not. **Both already have exactly one authored model.** What they cannot do is: + +1. **Reach it** — point a consumer at a tree that lives elsewhere in the repo. +2. **Scope it** — take the part of it this consumer cares about. + +Composition of multiple collections is a real need, but it is the *third* one. Building it first +(as FR-023 does) leaves both repos exactly as blocked as they are now. + +## 3. The load-order finding, and what it deletes + +The Java design assumed sources must be read in a correct sequence. That assumption no longer +holds anywhere in the codebase, and verifying it was the single largest simplification in this +design. + +- **Super-resolution is order-independent** — a pure function of the source set (#188). +- **The loader already discards the caller's order.** `MetaDataLoader._partitionOverlayLast` reads + every source, classifies each as base or overlay-only, and reorders them, with the comment + *"making the merge order-independent."* The ordered-list API takes an order it does not trust. + +Order still matters in exactly three places, and only one concerns the source list: + +| Where | Order-sensitive? | Concerns the source list? | +|---|---|---| +| Child order **within** a node — M:N reference direction, stored-proc argument binding, payload field order | Yes, load-bearing | **No.** Set by each file's own arrays at parse time. Untouched by this design. | +| Overlay precedence **across** sources | Yes | Yes — and the loader already *derives* it rather than trusting the caller. | +| Output determinism | Yes | Satisfied by a canonical sort over resolved source ids. Declared order not required. | + +**Therefore the declared source order carries no information the loader needs. Sources are a set.** + +**Order-independence is three layers, not one, and they are not redundant.** Implementation +confirmed this empirically, and the sharper statement belongs here because two readers in a row +conflated the layers and wrote a vacuous test as a result: + +1. **`resolveSources` canonicalizes.** It sorts resolved absolute paths, so in production the + loader never sees a permuted file list at all. +2. **The loader resolves CONTENT order-independently.** `_partitionOverlayLast` stable-partitions + overlay-only sources to merge last, so an overlay reaching a base declared in another file + resolves the same regardless of which arrived first. Disabling it throws `ERR_OVERLAY_NO_TARGET` + on half of the permutations of a two-file overlay set and silently drops the overlay's fields on + the rest. +3. **SIBLING ORDER of unrelated top-level nodes still follows load order, and that is *not* a + contract** — `canonicalSerialize` only ever promised attr-key alphabetization. A test that + demands byte-identical whole-tree serialization across permuted loader inputs is asserting a bar + the design never set, and it will fail on correct code. + +Layers 1 and 2 are pinned by `server/typescript/packages/sdk/test/order-independence.test.ts` — the +executable form of this statement. + +What this deletes, rather than adds: + +- No ordered-list semantics to specify, document, or port. +- No topological sort of dependencies, and **no cycle-detection error class** — a cycle in a set + union is not a thing. (`resolveExtendsOrder` in `sdk/src/workspace.ts` exists solely for this.) +- No diamond-dependency problem: two sources both contributing a shared model dedupe by identity. +- No serialization constraint on resolution: sources can be resolved concurrently, because nothing + needs to know its position before it resolves. + +It also matters for work that is explicitly *not* in this design. Runtime metadata sources — a +database, a registry — cannot guarantee row or stream order. And because resolution is a pure +function of the set, a set can be **added to and re-resolved**, which is the only coherent basis +for incremental or hot-reloading runtime metadata. Under ordered lists, "where in the sequence does +the new thing go?" has no good answer. This design banks that property without building on it. + +## 4. Design + +### 4.1 A collection is a set of sources + +A **metadata collection** is a named set of sources plus the metadata loaded from them. It is a +**tooling-config concept and never metamodel vocabulary** — every project surveyed in the prior art +keeps "what is a collection and where does it live" outside the modeled types. Consequences: +`registry-conformance` is unaffected, ADR-0023 provenance is unaffected, and no `expected-registry` +entry changes. + +Sources are declared as a JSON/YAML array for ergonomic reasons, but the array is **specified as a +set**: position is not meaningful, and a conformance gate enforces it (§4.7). + +### 4.2 Source kinds — a tagged union + +Following the prior art's tagged-union grammar rather than a prefix-disambiguated single string. +The single-string form (local paths must begin with `./`, git needs a `git::` prefix) buys terseness +at the cost of a grammar the parser owns and ambiguity as a live failure mode. A tagged union is +self-documenting, machine-checkable, and extensible without touching a parser — consistent with +ADR-0037's stated bias toward self-documentation over economy. + +```jsonc +"sources": [ + { "path": "../model-module/src/main/resources/metadata" }, // phase 1 + { "resource": "acme/model" }, // phase 1, JVM only + { "package": "@acme/common-model" }, // phase 2 (FR-023) + { "url": "https://…/model.tar.gz" }, // phase 3 + { "collection": "model" } // phase 3 (§4.8) +] +``` + +- **`path`** — a directory or file, relative to the declaring config file (never to ambient cwd — + the `--cwd` bug class in 0.20.1 is the precedent). Read **in place, never installed**, matching + every surveyed tool's treatment of local paths. +- **`resource`** — a classpath resource root. JVM-only; the Java port already implements it as + `model:resource:` and this design keeps that mechanism, only re-spelling its declaration. Other + ports reject it with a clear "not supported on this port" error rather than silently ignoring it. +- **`package` / `url` / `collection`** — later phases; the union shape is fixed now so they slot in + without a config migration. + +A source that does not resolve is an **error**, never a silent skip. Silent skip is how the +symlink-and-hope status quo fails today. + +**Vendoring falls out for free, and should be documented as a supported workflow.** Airgapped and +audit-constrained builds need the Go `go mod vendor` pattern — resolved dependencies committed into +the repo for hermetic builds. Because a `path` source is read in place and never installed, vendoring +is simply "copy the dependency into a directory and point a `path` at it." No mechanism is required; +what is required is saying so in the docs, rather than leaving enterprise adopters to discover it. + +### 4.3 Scope — package patterns, at output only + +**The collection loads in full. Scope applies to output, never to input.** + +Input-side subsetting is not merely tedious, it is wrong by construction: a partial file list can +fail to load because an `extends` target is missing, so the author must hand-maintain a transitive +closure. That is precisely what the 91-entry list is, and precisely why four files on disk sit in +an unresolvable "deliberate or forgotten?" state. Loading the whole collection is closure-complete +by definition. + +Scope is declared as **package patterns**: + +```jsonc +"scope": { + "include": ["acme::commerce::**", "acme::common::*"], + "exclude": ["acme::commerce::internal::**"] +} +``` + +**Semantics.** Patterns match a node's fully-qualified name. `*` matches exactly one package +segment; `**` matches any depth. Absent `include` means everything; `exclude` is applied after +`include`. An unparseable pattern is an error, not a non-match. + +**Two deliberate deviations from the shipped Java spelling**, both meeting the "only for good +reasons" bar: + +1. **Explicit `include`/`exclude` arrays instead of a single list with a `!` prefix.** The `!` + sigil is fatal in the YAML authoring front-end (ADR-0006): a leading `!` is YAML's tag + indicator, so every exclude would require quoting forever. That is a footgun, not a preference. +2. **`*` is one segment, `**` is any depth**, replacing Java's `*`-crosses-everything plus an `@` + escape for single-segment matching. The current behavior carries its own TODO in + `GeneratorUtil.createRegexFromGlob` admitting `::` is not enforced as a separator. Porting a + known bug to four more languages is worse than fixing it in one. + +Java's existing `` element keeps its current semantics unchanged; the new `scope` element +is a distinct key. No shipped pom breaks. + +**Why package patterns and not a predicate function.** TypeScript's per-generator `filter` is a +JavaScript function. It cannot be written in Python's `metaobjects.config.yaml`, C#'s CLI flags, or +a `pom.xml`, and it cannot be gated by any conformance corpus. Package patterns are strings and +port to all five config surfaces unchanged. The function filter is therefore **retained as-is, +unchanged, TS-only, documented as an escape hatch — and never the thing a cross-port feature +depends on.** Nothing is deprecated and no adopter migrates. + +Field evidence supports the demotion. The one adopter on a current release uses **zero** function +filters across nine generators. The adopter that uses three is seven minor versions behind, and its +own config comments describe exactly the defects fixed centrally in 0.21.5 (#248) — write forms +emitted for projections lacking an insert schema, hooks emitted for `object.value`, CRUD code +referencing exports abstract entities do not have. A user writing `filter: (e) => !e.isAbstract` is +compensating for a generator that should already know, which is a library bug report rather than a +config feature. Kind and shape predicates belong in the generator's central guards, where #248 put +them. + +### 4.4 Where scope attaches, including the DB-facing commands + +| Attachment | Applies to | Notes | +|---|---|---| +| Collection-level `scope` | Everything the consumer emits | A default for the consumer | +| Per-generator `scope` | That generator's output | Narrows the collection default | +| **Per-command `scope`** on `migrate` / `verify --db` | Which tables/views the command governs | **Required in phase 1** | + +Narrowing only: a generator or command scope **intersects** the collection scope and can never +widen it. Predictable, and it makes the collection-level declaration a real ceiling. + +The command-level scope is not a convenience. Generator scope does not reach `migrate` or +`verify --db`, where the **loaded model is the scope** — so "load everything" would otherwise take +a real adopter's worst standing hazard (a `--from-db` migrate proposing to drop tables it does not +model) and convert it from a discipline someone can follow into an automation nobody can. That +adopter already states the rule in prose in its own memo — *migrate owns one package tree's tables; +another tool owns the other's* — which is already a package pattern. This makes it declarative and +checkable: + +```jsonc +"migrate": { "scope": ["acme::platform::**"] } +``` + +Tables outside the scope are neither created nor dropped, and `verify --db` reports them as +out-of-scope rather than as drift. + +### 4.5 Precedence — a rule, not a position + +With order gone, overlay conflicts need a declared rule. Three cases, exhaustive: + +1. **Within one source set, base vs overlay-only** — unchanged. The loader's existing derived + partition (base first, overlay-only last) already handles it. +2. **Local vs dependency** — a `path` (or `resource`) source **wins** over a `package` source. This + is FR-023's "local overlays win" intent, expressed as a property of the source kind instead of a + position in a list. +3. **Two dependencies conflicting** — an **error**, not silent last-wins. Two independently + versioned packages declaring the same node non-overlay is a genuine ambiguity, and resolving it + by whichever happened to resolve first is the class of bug this whole design exists to remove. + +### 4.6 Where the declaration lives — one port-neutral file, five CLIs + +A polyglot repo breaks any design that treats the per-port config files as interchangeable +discovery targets. A Java consumer's configuration is a `pom.xml`; `migrate` and `verify --db` are +**Node-CLI-only** (ADR-0015). So the Node CLI must operate on a model declared by a Maven module, +and would find nothing if it looked only for `metaobjects.config.ts`. + +Two different questions are being conflated, and they already have two different homes: + +| Question | Home | Read by | +|---|---|---| +| Where does metadata come from, and what is in scope? | **`.metaobjects/config.json`** (port-neutral JSON) | **all five CLIs** | +| How is code generated here? | `metaobjects.config.ts` / `metaobjects.config.yaml` / pom `` | that port only | + +This is not a new split — it is the one CLAUDE.md already documents ("`.metaobjects/config.json` +(JSON) — static project state. Parseable by non-TS tooling"). It is also already scaffolded: every +`meta init` project carries `"sources": []` in that file today, empty and inert. Phase 1 fills the +slot that already exists. + +**One gap must close in phase 1.** A JVM-rooted adopter has no `.metaobjects/config.json` at all — +only agent-context files, because it was scaffolded with `agent-docs` rather than `meta init`. The +file is therefore port-neutral in theory and TS-scaffolded in practice. **Every port's CLI must be +able to create and read it**, or the neutral file is neutral in name only. + +### 4.6.0 One authority, and `metaobjects/` is only a default + +**No adopter ever needs a directory named `metaobjects/`.** It is the default value of `sources` +when the key is absent or empty — never a requirement, and never assumed by any code path. + +The rule: **`sources` is the single authority on where metadata lives, and everything that needs to +find metadata reads it.** Today that is false in TypeScript in ten places, which is the concrete +phase-1 work item: + +| Site | Kind | Phase 1 | +|---|---|---| +| `cli/commands/docs.ts` (×3), `export.ts`, `gen.ts` | read | route through resolved `sources` | +| `cli/commands/prompt-snapshot.ts` | read | route | +| `cli/index.ts` — the "is this a MetaObjects project?" probe | read | route | +| `cli/lib/detect-stack.ts` — concern detection | read | route | +| `sdk/memory.ts` (×2) — the loader entry itself | read | route | +| `cli/commands/init.ts` (×2) | **write** | **keep the literal** — scaffolding the default is the one place it belongs | + +`prompt-snapshot.ts` was missing from the first draft of this table, which is why nothing scheduled +it; it is listed now because the ports plan is written from this table and would otherwise inherit +the omission in four more languages. It matters more than its size suggests: `--check` is a drift +GATE, so a project declaring `sources` elsewhere would gate against a stale `metaobjects/` rather +than fail. + +**Python is already the reference implementation of this shape**, not a laggard: its project config +reads `metadata` from the config file with the directory name as a *fallback* +(`raw.get("metadata", DEFAULT_METADATA_DIR)`). It needs widening from one string to a set, not +rearchitecting. C# takes the directory as a positional argument, which is configurable by a +different route. **TypeScript is the outlier that hardcodes.** + +`detect-stack.ts` is the load-bearing one and the least obvious. It scans for `requirement.` +markers to derive concern tokens for agent-context scaffolding; a project pointing `sources` +elsewhere gets a **silent false**, scaffolding the wrong agent docs with no error. That is the same +"two code paths disagree about where metadata is" failure as the nested-symlink divergence in §8, +from the same root cause — so routing every read through one authority closes both. + +### 4.6.1 Discovery — nearest ancestor, explicit override, no auto-discovery + +Running a CLI inside an app must find that app's configuration. + +- **Walk up from cwd** for the nearest `.metaobjects/config.json`. Nearest wins. Per-port + generator config is then read from that same directory. +- **That file is the ONLY project marker.** A directory that merely *holds* metadata is not a + project boundary. Where metadata lives is the `sources` key's answer, and the default + directory name is only that key's default *value* — so stopping the walk on a directory of + that name would put a second definition of "where metadata lives" back into the toolchain, + which is the exact duplication §4.6 exists to remove. It would also be wrong on its own + terms: a project whose config points `sources` at a sibling module has no such directory at + all, and one that has both would be governed by whichever the walk noticed first. A + subdirectory that should own its metadata declares a config — `meta init` writes one. +- **Stop at a repository boundary** (`.git`) or the filesystem root, so a monorepo can never + silently adopt a parent checkout's configuration. +- **Explicit override wins** — the existing `--cwd` / `-C` flag and project-root positional are + unchanged and take precedence over discovery. +- **Collections are never auto-discovered.** No globbing for directories that look like metadata + homes. A collection exists only where a config names one — Go's stance rather than Cargo's, + because a polyglot repo has many directories that merely *look* like collections, and silent + membership is the hardest failure to debug. + +For Java and Kotlin, discovery is a non-issue for *codegen* and stays that way: the Maven reactor +already runs the plugin per module with that module's own configuration. It is emphatically not a +non-issue for the Node CLI operating on those same modules, which is what §4.6 exists to solve. + +**Two failure modes get explicit, useful errors rather than silence:** + +- **Invoked at a repo root that declares no `sources`** — error listing the consumers discovered + beneath it ("did you mean one of…"). This requires a downward scan, but **for the error message + only, never for resolution**, so the no-auto-discovery rule is preserved. +- **Invoked inside a collection** (a directory that is metadata, not a consumer of it) — a distinct + error saying so, rather than an empty load. + +**Existing single-directory projects are unaffected.** One config at the root, one implicit +`{ "path": "metaobjects" }` source, no scope — byte-identical output. + +### 4.6.2 Schema ownership is not codegen consumption + +A polyglot repo has **many codegen consumers and at most one schema owner per database.** In the +larger adopter, six consumers read one model over one Postgres; if each declared a `migrate` scope, +six partial migrations would result — worse than today. The two-rail adopter has the same problem +already and names it in its own memo as needing an explicit ownership rule. + +The marker already exists and does not need inventing: **whoever holds `.metaobjects/migrations/` +and the schema snapshot owns the schema.** So: + +- A `migrate` block (§4.4) is valid only in a consumer that holds a ledger. +- A second consumer running `migrate` against the same database is detectable through the ledger + rather than left to discipline. +- `verify --db` may run from any consumer, reporting out-of-scope tables as out-of-scope rather + than as drift. + +### 4.7 Conformance + +Two new corpora, plus one gate that is the linchpin of the whole design. + +1. **Order-independence gate (the linchpin).** The same source set is loaded in N permutations and + the canonical serialization must be **byte-identical** across all of them, in all five ports. + Without this, set semantics is an aspiration that decays the first time someone adds an + order-sensitive code path. With it, the property is enforced rather than believed. + **Corrected during implementation — see §3's three-layer statement:** whole-tree byte-identity + is too strong a bar, because sibling order of unrelated top-level nodes legitimately follows + load order and was never a contract. The gate asserts layers 1 and 2 (identical `resolveSources` + output across permutations; identical resolved CONTENT across permuted loader inputs), which is + the property this item was reaching for. +2. **Scope-pattern corpus.** A matrix of patterns × fully-qualified names → expected match/no-match, + byte-matched across all five ports. This is what stops `*` and `**` from meaning five different + things — the failure mode that produced the `like`/`ILIKE` divergence. +3. **Discovery** is filesystem behavior and stays per-port, not corpus-gated. + +New error codes register in all three ledgers (TS `errors.ts` exact-bidirectional, Python +`errors.py` superset, Java `ErrorCode.java`): `ERR_SOURCE_UNRESOLVED`, +`ERR_SOURCE_KIND_UNSUPPORTED`, `ERR_SCOPE_PATTERN_INVALID`, `ERR_COLLECTION_NOT_FOUND`, +`ERR_DEPENDENCY_DECLARATION_CONFLICT`. + +### 4.8 What ships when + +**Phase 1 — the spine (releasable on its own; unblocks both adopters).** +`sources` with `path` and `resource`; `scope` with `include`/`exclude` and `*`/`**`; +nearest-ancestor discovery; load-everything; per-command scope for `migrate`/`verify --db`; the +order-independence and scope-pattern corpora. This alone turns 91 hand-maintained `` lines +into one path plus one pattern, and deletes the symlinks. + +**Phase 2 — package sources (FR-023, re-scoped).** The `package` resolver per ecosystem, the +package manifest, and per-package provenance attribution. Spike-validated for NuGet (§5). + +**Phase 3 — remote and named collections.** `url` sources with the pin-and-cache discipline every +surveyed tool has; `collection` references via an **optional** root file that names shared +collections and nothing else (§6). + +**Out of scope.** Database and other runtime sources. Ruled a runtime-metadata concern, not a +build-time one — consistent with every system surveyed, where reading schema from a live store +serves a running application and never a build. It gets its own FR. + +### 4.9 Naming + +The config key is **`sources`**, matching the Java loader element and the (currently dead) key +already present in `sdk/src/config.ts`. This collides by name with the `source.*` metamodel node +type, which was flagged as a concern worth recording. The collision is judged acceptable: the two +never appear in the same file — `source.rdb` is a node inside a metadata document, `sources` is a +key inside a tooling config — and the Java port has carried both for years without incident. + +The decisive argument is that **the key is already scaffolded into every project**: `meta init` +writes `"sources": []` into `.metaobjects/config.json`, and real adopter repos carry it today. The +slot exists, is empty, and is waiting; renaming it now would orphan it in every scaffolded project +for no semantic gain. If review disagrees, `metadataSources` is the alternative and costs nothing +but verbosity plus a scaffold migration. + +## 5. Spike results + +**Spike 1 — a code-free NuGet package can be resolved by a non-MSBuild CLI. Confirmed.** A real +`.nupkg` carrying a `metaobjects/` tree was packed and consumed from a `PackageReference` project. +Findings: `contentFiles` copies nothing useful and is the wrong mechanism; a `build/*.targets` file +correctly exposes an MSBuild property pointing into the extracted package; and — the result that +matters — `obj/project.assets.json` carries the package folder root, the library's relative path, +and a **complete file listing including the metadata files**, so a plain CLI resolves the tree by +joining two strings and reading JSON. Two warts, both minor: packing a code-free package emits +warning `NU5128` (suppressible), and `project.assets.json` only exists after a restore — which is +the same explicit-fetch precondition every surveyed tool has. + +This retires the main technical objection to per-ecosystem publishing. It does not settle the +question (§7). + +**Spike 2 — a root config declaring N collections and N consumers fails structurally.** The Buf +precedent does not transfer, and the reason is worth recording: Buf can put everything in one root +file because **Buf owns its entire config surface**. MetaObjects does not — the build tool does. A +TypeScript consumer's config holds executable generator wiring; a Java consumer's lives in its +`pom.xml`. A root config declaring consumers would have to duplicate or override both. What +survives is per-consumer declaration plus nearest-ancestor discovery, adding two keys to files that +already exist. + +## 6. Deferred: named collections + +Per-consumer declaration repeats the collection path once per consumer — six times in the larger +adopter. The fix is an **optional** root file that declares *only* where shared metadata lives, +never generator wiring and never output: + +```jsonc +// /.metaobjects/collections.json (optional) +{ "collections": { "model": { "sources": [{ "path": "model-module/src/main/resources/metadata" }] } } } +``` + +Consumers then write `{ "collection": "model" }`. Strictly additive, imports none of Shape A's +failure, and changes no semantics — so it can land whenever a repo actually feels the repetition +rather than on speculation. + +## 7. Open questions for review + +1. **Per-ecosystem publishing vs OCI.** FR-023 proposes the same code-free artifact in four + registries. No surveyed peer does this: CUE explicitly rejected it for OCI on polyglot grounds, + Buf built its own registry, Smithy stayed single-ecosystem. Spike 1 shows the mechanism works, + and the existing four-registry lockstep release machinery is an advantage none of those projects + had — but four artifacts of identical bytes means four version numbers, four resolvers, four + caches and four chances to drift. **This should be an ADR with an argued decision, not a default + inherited from FR-023.** It does not block phase 1. + + **The polyglot case sharpens this from a preference into a requirement.** *Within* one repo every + source is a `path`, so no registry is involved. But a shared model consumed **across** repos by a + polyglot consumer set must be reachable from each ecosystem: a `resource:` classpath source is + unreachable to the Node CLI, and an npm package is unreachable to Maven. So a cross-repo shared + model needs publication to **every ecosystem that consumes it**, or a single ecosystem-neutral + channel (OCI). "Publish to one registry and let others cope" is not an available option — which + is exactly the trade-off CUE resolved by leaving per-ecosystem registries behind. + + **A counterweight that cuts the other way, and belongs in the ADR.** Enterprises already run + internal mirrors of npm, Maven, PyPI and NuGet (Artifactory, Nexus, Azure Artifacts), with + scanning, approval and supply-chain policy already attached to them. Publishing to those four + means an adopting enterprise's **existing** infrastructure works unchanged; OCI generally + requires registering a new artifact type and new policy to go with it. This is an argument about + the *consumer's* infrastructure rather than the publisher's convenience, which is why neither + CUE's reasoning nor the initial framing of this section accounted for it. +2. **`sources` vs `metadataSources`** (§4.9). +3. **Scope on `verify --codegen`.** Drift detection compares generated output to metadata; if scope + narrows what is generated, drift must be evaluated within the same scope or every out-of-scope + file reads as drift. Believed straightforward; call it out so it is not discovered late. + +## 8. Risks and honest costs + +- **~~This repository cannot dogfood the feature.~~ REVISED — it can, and it should.** This repo is + itself the shape the design serves: five ports, a Maven reactor, ~20 TypeScript packages, Python, + C#, client packages. Once `sources` is the authority (§4.6.0), a first-party shared collection can + live **wherever makes sense for this repo** — no root `metaobjects/` required — and be consumed by + each port's integration tests via a `path` source plus a `scope`. That exercises reach, scope and + discovery across all five languages in this repo's own CI, which is exactly the phase-1 surface. + **But it does not close the risk**, because this repo's consumers are *ports*, not applications: + it proves the mechanics, never the product path (codegen into a running app against a database). + So an external smoke test against a real multi-consumer layout remains a phase-1 release gate — + demoted from the only gate to the second one. + +- **A layout question this repo has not had to answer before.** A code-free metadata package is + neither server-side nor client-side, so the "deployment target → language → framework" rule in + CLAUDE.md does not place it. Recommend a new top-level sibling — `spec/` holds the metamodel (the + language), so a `model/` would hold models expressed in it (the content). Small, but it should be + decided rather than defaulted. +- **Load-everything is O(collection), not O(scope).** Each consumer loads the whole collection even + when it emits a fraction. At current adopter sizes (~120 files) this is not measurable, but it is + a real asymptote and should be stated rather than discovered. +- **Discovery can surprise.** Walking up to find a config is the least surprising behavior in + developer tooling *and* a new way to pick the wrong file. The `.git` stop condition and the + explicit-override precedence are the mitigations; both need tests. +- **Two scoping mechanisms coexist in TypeScript** — package patterns and the retained function + filter. Documentation must be unambiguous that only the former is a cross-port concept, or the + next cross-port feature will be built on the one that cannot port. +- **Java carries two filter spellings** — legacy `` with its existing semantics, and the + new `scope`. Intentional, to avoid breaking shipped poms, but it is two things to explain. + +- **Toolchain version skew across ports is a polyglot hazard nothing currently checks.** A repo + whose consumers span five ports pins five toolchains, and two consumers loading the same + collection under different loader versions can legitimately disagree about it. The two-rail + adopter already names this in its own memo ("one home = one pin") after paying for a stale-pin + incident. The lockstep release policy makes agreement *possible* — all four registries share + `minor.patch` — but nothing enforces it inside a consuming repo. Recommend a warning-level check + once more than one consumer resolves the same collection; not a phase-1 blocker, but it should + not be discovered by an adopter. + +- **A pre-existing symlink inconsistency this work should absorb.** Surfaced while investigating + the adopter workaround, and verified empirically rather than reasoned about. A **top-level** + symlinked metadata directory *is* followed by both code paths. But for a **nested** symlinked + subdirectory the two disagree: `detect-stack.ts` walks with `readdirSync(…, {withFileTypes:true})` + and a `Dirent` for a symlinked directory reports `isDirectory() === false`, so it does not + descend — while `sdk/src/memory.ts` walks with `stat`, which follows, so the loader does. **The + same tree is one shape to the loader and a different shape to stack detection.** Phase 1 removes + the *need* for symlinks in the adopters that use them, but it does not fix this, and the + divergence outlives them. Worth folding into phase 1 rather than leaving as a latent trap. + (Note this corrects the framing in the handoff that motivated this work, which described the + top-level symlink as unfollowed.) + +## 9. Release shape + +Phase 1 is **additive** — new config keys, no change to any existing single-directory project's +output. It introduces a new capability adopters opt into deliberately, so it is a **MINOR** under +ADR-0035 Amendment 1's consumer-impact test rather than a patch: pre-1.0 caret ranges make a minor +a deliberate adoption, which is the correct gate for a change to how metadata is located. + +## 10. Adjacent capabilities — surveyed, deferred, filed + +A pass over the surveyed projects for enterprise capabilities MetaObjects lacks. **None is required +for phase 1**, and each was checked against the codebase before being called a gap. Filed so they +are not re-derived: + +| # | Capability | Why not phase 1 | +|---|---|---| +| #299 | **Producer-side access control** (`private`/`protected`/`public`, per dbt; `@internal` + transform, per Smithy). We have consumer-side scope only — everything in a collection is visible to everyone who loads it. | Metamodel vocabulary (MINOR); phase 1 is config-only. Becomes load-bearing when *cross-team* sharing starts, i.e. with package sources. Phase 1 must not preclude it. | +| #300 | **Breaking-change detection** against a baseline revision. Highest enterprise value in this set — it is the difference between a shared model that can evolve and one nobody dares touch. | Downstream of shared collections existing. Note we already have two-thirds: `migrate` gates schema-breaking changes behind `--allow` tokens, `verify --codegen` covers code drift; what is missing is metadata-vs-metadata across revisions. | +| #301 | **Dependency override** (Go's `replace`) for running a patched shared model. | A `path` source *is* an override while only `path` exists. Needed once `package` lands. | +| #302 | **Severity levels + suppressions** (per Smithy). | Not required by phase 1's new errors — but note this gap has already forced two design compromises (object coverage shipped as a warning because it would convict a project's first `verify`; `@verifiedBy` had to warn rather than convict on an unrecognised convention). | +| #303 | **Ownership metadata** (dbt groups carry owners). | May be adequately served by the `attr.properties` bag; run ADR-0037 before adding vocabulary. | +| #304 | **`meta fmt`** — the canonical serializer already exists in all five ports and is not exposed as a command. | Cheap, but orthogonal. | +| #305 | **Enforce `@deprecated`** — registered in all five ports, read by nothing. | Orthogonal; improves markedly once #302 lands. | +| #306 | **`meta why `** — per-node source provenance query. | Attribution is *already* tracked and surfaced in loader diagnostics, so phase 1's "which source did this come from?" need is met by error messages. A query is convenience on existing data. | + +**Deliberately not taken**, to prevent later scope creep: dbt's cross-project references depend on a +stateful metadata service (we should not build a service); Terraform's state model does not apply; +CUE's unification is a different language paradigm; and Buf's own registry is precisely what riding +existing ecosystems avoids. + +## 11. What gets deleted + +- `resolveExtendsOrder`'s topological sort and its cycle-detection error path + (`sdk/src/workspace.ts`) — meaningless under set union. +- The `package.meta.json` workspace mechanism — structurally JS-only (it recognizes a workspace + root solely by `pnpm-workspace.yaml` or `package.json` workspaces, so a Java + Python repo + silently falls through to the single hardcoded directory). It cannot be grown into this. + **Removal is not free:** the file is scaffolded by `meta init` and present in real adopter repos, + though always as an inert `{name, version, extends: []}` with an empty `extends`. Since nothing + populates `extends`, narrowing it to a no-op and removing it in a later major is the safe path — + deleting it in phase 1 would edit adopters' repos for no functional gain. +- The dead `sources` key's *emptiness* in `sdk/src/config.ts` — the key itself is kept and given + the real schema (§4.9). +- Two directory symlinks and ~158 hand-maintained file paths, in adopter repos. diff --git a/docs/superpowers/specs/2026-08-17-metadata-source-resolution-prior-art.md b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-prior-art.md new file mode 100644 index 000000000..19019ffc2 --- /dev/null +++ b/docs/superpowers/specs/2026-08-17-metadata-source-resolution-prior-art.md @@ -0,0 +1,384 @@ +# Metadata source resolution — prior art + +_Status: RESEARCH (evidence for a design; contains no decisions)._ +_Date: 2026-08-17._ +_Feeds: the metadata-collection / source-resolution design (supersedes the scope of +`2026-06-11-fr-023-metadata-packages-design.md`, which is one resolver within it)._ + +## Why this document exists + +MetaObjects needs to answer four questions it has never answered uniformly across its +five ports: + +1. **What is a metadata collection?** Today a run loads exactly one directory (`metaobjects/`) + in four of five ports. A monorepo with several apps sharing a common model cannot express + itself. +2. **How does a tool know which collection it is working in?** A CLI invoked at a monorepo + root sees only the root's metadata, which is the wrong answer for an app three directories + down. +3. **How is a shared model distributed and depended upon** across repos and across languages + (npm / PyPI / NuGet / Maven — or something else)? +4. **Can a source be remote** (a URL, an OCI artifact, a database) and if so, at build time, + at runtime, or both? + +Every one of these has been solved, several times, in public, by projects with the same +shape: a declarative model, a toolchain that reads it, multiple languages downstream, and +users with monorepos. This document records what those projects actually do, so the design +argues from evidence rather than from first principles. + +## Sourcing and IP hygiene + +Rules followed while compiling this: + +- **Open-source projects only.** Every project surveyed ships its tool under an OSI-approved + license, and every behavior described is from that project's **public documentation**. +- **Behavior and public config grammar only.** No source code was read, copied, or adapted. + Descriptions are written from scratch in our own words; nothing is quoted at length. +- **Hosted/commercial components are explicitly excluded.** Several of these projects pair an + open-source CLI with a commercial hosted registry. Only the open-source CLI's *local + configuration and resolution behavior* is recorded here. The hosted services' internals, + APIs, and pricing models are out of scope and were not investigated. +- **Every claim carries a URL.** Claims marked _(background)_ are widely-known, long-standing + toolchain behavior (the Java classpath, `sys.path`, nearest-ancestor config discovery) that + was not re-verified against a citation in this pass; they are included for completeness of + the pattern and should not be treated as researched findings. +- **Licenses are noted per project.** Where a project's license changed (Terraform), that is + called out along with its OSS fork, so nothing here is mistaken for guidance to depend on a + non-OSS artifact. + +This document describes *patterns*, which are not protectable. It is a survey, not a +derivative work. + +--- + +## Part 1 — The seven recurring patterns + +### P1. The ordered root list ("include path") + +The oldest and most durable shape: a tool is given an **ordered list of roots**, and a +reference is resolved by trying each root in order until one matches. + +- **protoc** takes one or more `--proto_path` / `-I` roots. An `import` is resolved relative + to each root **in order, first match wins** — the documented idiom being to put a local + tree ahead of a vendored tree so a local override shadows a dependency. + ([protoc reference](https://protocolbuffers-protobuf-45.mintlify.app/tooling/protoc), + [Go Protobuf Tips](https://jbrandhorst.com/post/go-protobuf-tips/); protobuf is BSD-3-Clause) +- The **Java classpath**, Python's **`sys.path`**, and Ruby's **`$LOAD_PATH`** are the same + primitive: an ordered sequence of roots, resolved left to right. _(background)_ + +**Why it matters here:** MetaObjects' Java port already implements exactly this — the Maven +plugin's loader parameter takes a `sourceDir` plus an **ordered list** of sources. The other +four ports collapsed it to a single directory. This is not a new idea to invent; it is an +existing idea to propagate. + +**The subtlety worth stealing:** first-match-wins over an ordered root list, and +last-writer-wins overlay merge (MetaObjects' existing semantics), are *different* composition +rules. protoc shadows a whole file; MetaObjects merges node-by-node. A design that borrows the +ordered list must say explicitly which rule applies at which layer. + +> **CORRECTION (added on review — this pattern does NOT apply, and leading with it was a +> mis-generalization).** Every example in P1 is a **shadowing** mechanism: order encodes +> precedence and the losing file is discarded entirely. MetaObjects *merges*. Adopting an ordered +> include path would therefore import order-sensitivity that the engine has already engineered +> away — super-resolution is a pure function of the source *set*, and the loader already discards +> the caller's declared order in the one place it matters (it reads every source, classifies each +> base-vs-overlay, and reorders). P1 is the oldest pattern here and the **least** applicable. See +> P8, which is what the dependency-management prior art actually shows. + +### P2. Collection identity, and a workspace of N collections + +Once there is more than one collection, each needs a name, and something has to describe the +set. + +- **Buf v2** merged what were previously two files into one: a single `buf.yaml` at the + workspace root declares **multiple modules**, each with its own directory and its own + lint/breaking-change settings, while **external dependencies and the lock file are shared + across the whole workspace**. Critically, **dependencies *between* modules in the workspace + are not declared** — the tool infers them from the module set. One publish command covers + every module in dependency order. + ([modules and workspaces](https://buf.build/docs/cli/modules-workspaces/), + [v2 migration guide](https://buf.build/docs/migration-guides/migrate-v2-config-files/); + the Buf CLI is Apache-2.0 — [repo](https://github.com/bufbuild/buf)) +- **Cargo** defines a workspace via a `[workspace]` section in a `Cargo.toml`, which may be a + "virtual" manifest (workspace only, no package of its own) or a real one (both). Members are + listed as glob patterns, and the workspace shares one lock file and one build output + directory. ([Cargo workspaces](https://deepwiki.com/rust-lang/cargo/2.2-workspaces); + Cargo is MIT OR Apache-2.0) +- **Go workspaces** take the opposite stance on discovery: a `go.work` file lists module + directories by **explicit relative path**, and Go deliberately does **not** auto-discover + modules — you add them with an explicit command. Absent a `go.work`, the workspace is simply + the single module containing the current directory. + ([Go modules reference](https://go.dev/ref/mod); Go is BSD-3-Clause) + +**The design fork this exposes:** glob-based auto-discovery (Cargo, Buf) versus explicit +enumeration (Go). Go's rationale — no surprises, no accidental membership — is the stronger +argument in a polyglot repo where a stray directory could otherwise be swept into a build. + +### P3. Contextual discovery — nearest ancestor, plus an explicit override + +Every tool in this space eventually needs to answer "which project am I in?" and they all +converge on the same two-part answer. + +- **graphql-config** supports both shapes and says so explicitly: either multiple named + `projects` in one root config, **or** one config file per subdirectory, where the config + file's location defines a scope for its whole subtree. Their framing is the clearest + statement of the principle — a config file marks a module root the same way `package.json` + does, and editor tooling picks the config **closest in the directory hierarchy** to the file + being worked on. ([graphql-config usage](https://the-guild.dev/graphql/config/docs/user/usage); + graphql-config is MIT) +- **dbt** defaults to looking for its project file in the current working directory **and its + parents**, with an explicit `--project-dir` flag (and an environment variable) to override. + ([dbt_project.yml reference](https://docs.getdbt.com/reference/dbt_project.yml); + dbt-core is Apache-2.0) +- `tsconfig.json`, `.editorconfig`, `.gitignore`, and `package.json` all use nearest-ancestor + resolution. It is the least surprising behavior in developer tooling. _(background)_ + +**The consistent shape: walk up from cwd to find the nearest collection root; allow an +explicit flag to name one; allow a root config to enumerate several.** No surveyed project +requires the user to pass a path on every invocation, and none auto-detects without an escape +hatch. + +### P4. How a source is spelled — three grammars + +Three distinct approaches to writing down "where this dependency comes from": + +- **Prefix-disambiguated single string (Terraform).** One `source` string covers local paths, + a module registry, Git, HTTP, and object storage. The kinds are told apart by *syntax*: a + local path **must** begin with `./` or `../` (which is what distinguishes it from a registry + address), Git sources carry a `git::` prefix, registry addresses use a + `namespace/name/provider` shape. Local paths are explicitly *not* "installed" — they are used + in place. ([module sources](https://developer.hashicorp.com/terraform/language/modules/sources)) + **License note:** Terraform moved to BUSL-1.1 in 2023. Only its publicly documented + configuration grammar is described here; the MPL-2.0 fork **OpenTofu** carries the same + grammar and is the OSS artifact to reference if this pattern is adopted. +- **Tagged union (dbt).** Dependencies are declared as a list where each entry names its kind + by key — a registry package, a Git repo, or a **local path**. dbt's documentation + specifically recommends **local packages as the monorepo answer**: several projects nested in + subdirectories, combined for coordinated development and deployment. + ([dbt packages](https://docs.getdbt.com/docs/build/packages)) +- **Ecosystem coordinates (Smithy).** `smithy-build.json` declares model dependencies as + **Maven GAV coordinates** plus repository URLs, and the CLI resolves them with the actual + Apache Maven dependency resolver. Shared models are published *inside a JAR* via a dedicated + packaging plugin that adds the model files and build metadata to the jar. + ([smithy-build.json](https://smithy.io/2.0/guides/smithy-build-json.html), + [Gradle plugins](https://smithy.io/2.0/guides/gradle-plugin/index.html); Smithy is Apache-2.0) + +**Smithy is the closest precedent for MetaObjects' instinct** — a schema-first, multi-language +codegen tool that resolves its *model* dependencies through an existing language package +manager rather than inventing distribution. Note what it did **not** do: see P5. + +**The trade-off:** a single prefix-disambiguated string is terse but forces the parser to own a +grammar and makes ambiguity a real failure mode (Terraform needs the `./` rule precisely +because of it). A tagged union is verbose but self-documenting, machine-checkable, and +extensible without touching a parser — which matches how MetaObjects already treats its own +config (`ADR-0037`'s bias toward self-documentation over economy). + +### P5. Distribution channel — the three-way split, and a notable negative + +This is where the surveyed projects disagree most sharply, and the disagreement is informative. + +- **Own registry (Buf).** Built a dedicated schema registry; the CLI's `deps` name modules in + it, and every commit is content-addressed by a cryptographic manifest digest recorded in the + lock file. ([dependency management](https://buf.build/docs/bsr/module/dependency-management/)) + The hosted registry itself is a commercial service and is out of scope here; what is relevant + is that Buf chose *not* to ride existing language registries. +- **Existing language registry, one ecosystem only (Smithy).** Uses Maven — and only Maven — + even though Smithy generates code for many languages. The model artifact is a JAR regardless + of which language you generate. +- **OCI registries (CUE).** CUE's module system is built on **OCI registries** rather than + ecosystem-specific ones. The stated reasoning is directly on point for a polyglot standard: + nearly every deployment already has an OCI registry available, the protocol is HTTP-based and + simple enough to implement a custom server against, and it is an open standard — so a single + artifact serves every language instead of N per-ecosystem copies of the same bytes. + ([CUE modules](https://cuelang.org/docs/reference/modules/), + [custom module registry](https://cuelang.org/docs/tutorial/working-with-a-custom-module-registry/), + [modules design proposal](https://github.com/cue-lang/proposal/blob/main/designs/modules.v3/2939-modules.md); + CUE is Apache-2.0) + +**The negative finding, stated plainly: no surveyed project publishes the same code-free +schema artifact to four language registries.** Every one either built its own registry, picked +a single ecosystem, or moved to OCI. CUE faced precisely MetaObjects' situation — a polyglot +declarative language needing cross-language model reuse — and explicitly rejected the +per-ecosystem approach. + +This does not make the four-registry plan wrong. MetaObjects already publishes to all four +registries in lockstep, so the release machinery exists and the marginal cost of a fifth +code-free artifact per registry is lower here than it would be for a greenfield project. But +it does mean the plan should be an **argued decision** rather than an assumption, and the +argument has to address what CUE's reasoning gets right: four artifacts of identical bytes +have four version numbers, four resolvers, four caches, four lockfiles, and four opportunities +to drift. + +### P6. Pinning and reproducibility + +Every surveyed project separates **declaration** from **resolution**, and records the +resolution. + +- **Buf** pins each dependency in a lock file by content-addressed digest, not merely by + version. ([dependency management](https://buf.build/docs/bsr/module/dependency-management/)) +- **Cargo** shares one lock file across the entire workspace. + ([workspaces](https://deepwiki.com/rust-lang/cargo/2.2-workspaces)) +- **Terraform** and **dbt** each have an explicit install/fetch step separate from use; dbt + vendors resolved packages into a local directory. ([dbt packages](https://docs.getdbt.com/docs/build/packages)) +- Local-path sources are the documented exception in both Terraform and dbt: they are **not + installed**, they are read in place. + +**The pattern: remote sources get an explicit fetch step and a pinned record; local sources +skip both.** No surveyed tool silently fetches a remote dependency during a normal build. +This has a direct consequence for MetaObjects: a URL source that is read at load time, on every +`meta gen`, with no lock and no cache, is a shape nobody in this space ships. + +### P7. Build-time versus runtime is a hard boundary + +Two entirely separate worlds, and no surveyed project blurs them. + +- **Build-time** distribution (everything in P5) resolves files onto disk before codegen runs. +- **Runtime** schema access is a different product category — a schema registry service + queried over HTTP by a running application, addressing artifacts by group/id/version and + returning the schema document. **Apicurio Registry** (Apache-2.0) is the open-source + reference: a REST interface where a client fetches a specific artifact version at runtime. + ([Apicurio introduction](https://www.apicur.io/registry/docs/apicurio-registry/3.1.x/getting-started/assembly-intro-to-the-registry.html), + [artifact reference](https://www.apicur.io/registry/docs/apicurio-registry/3.3.x/getting-started/assembly-artifact-reference.html)) + +**Relevance to "sourcing from a DB":** in every surveyed system, reading schema from a live +service or store is a **runtime** capability serving a running application — not a build-time +codegen input. A database as a *codegen* source would be novel, and novelty here is a cost: +it breaks reproducible builds (the source can change between two builds of the same commit) +unless paired with the P6 pin-and-cache discipline. A database as a *runtime* source is +well-trodden and is a different feature with different requirements. + +### P8. Nobody makes the user declare load order + +_Added on review, and it reverses P1's framing._ + +Re-read for order-sensitivity rather than for structure, the dependency-management prior art is +unanimous: **the user declares a SET, and the tool derives whatever order it needs.** + +- **Buf** is the most explicit: dependencies *between* modules in a workspace are deliberately not + declared, because the tool infers them from the module set — and a single publish covers every + module in the right dependency order, computed rather than written down. + ([modules and workspaces](https://buf.build/docs/cli/modules-workspaces/)) +- **Go**, **Cargo**, **dbt** and **Terraform** all take an unordered dependency declaration and + compute the graph. Nobody hand-sorts a `go.mod`, and dbt builds its DAG from model references + rather than from list position. ([Go modules](https://go.dev/ref/mod), + [Cargo workspaces](https://deepwiki.com/rust-lang/cargo/2.2-workspaces), + [dbt packages](https://docs.getdbt.com/docs/build/packages)) +- **Smithy** hands its coordinates to the Maven resolver, which owns ordering entirely. + +The only ordered-list examples in this document (P1) are shadowing mechanisms, where order *is* the +precedence rule rather than a load sequence. + +**Implication:** a config schema that asks an author to sequence sources is asking for information +no surveyed tool requires and this engine does not consume. Precedence, where it is genuinely +needed, should be expressed as a **rule attached to a source** ("a local source wins over a +dependency") rather than as a position in a list — which also makes diamond dependencies, parallel +resolution, and incremental addition fall out for free. + +### P9. Scoping is done with patterns, not predicates + +Every surveyed tool that scopes a large model scopes it with **declarative string patterns** — +never a callback. Buf modules take path `excludes`; Smithy's build config filters models +declaratively; dbt selects with a string selector syntax. + +MetaObjects' own Java port has carried this since long before this survey: `GeneratorUtil` +implements include/exclude patterns (a `!` prefix marks an exclusion) glob-matched against a node's +fully-qualified name, with `@` matching exactly one package segment. + +**Implication:** a predicate *function* — TypeScript's per-generator `filter` — cannot be expressed +in a YAML config, an XML pom, or a CLI flag, and cannot be gated by any cross-language corpus. It +is therefore unsuitable as a cross-port primitive regardless of its ergonomics in the one port that +can express it. + +--- + +## Part 2 — Evidence table + +| Project | License | Ordered roots | Multi-collection config | Contextual discovery | Dep grammar | Distribution | Pinning | +|---|---|---|---|---|---|---|---| +| protoc | BSD-3-Clause | **yes**, first-match-wins | — | — | — | — | — | +| Buf CLI | Apache-2.0 | — | **one config, N modules**, shared deps, intra-workspace deps inferred | workspace root | module refs | own registry | digest lock | +| Cargo | MIT OR Apache-2.0 | — | `[workspace]` + member globs | walk up | coordinates | crates.io | one workspace lock | +| Go modules | BSD-3-Clause | — | `go.work`, **explicit paths, no auto-discovery** | single module containing cwd | module paths | VCS-addressed | `go.sum` | +| graphql-config | MIT | — | `projects` map **or** per-subtree config | **nearest ancestor** | — | — | — | +| dbt-core | Apache-2.0 | — | local packages = the monorepo answer | walk up + `--project-dir` | **tagged union** (registry/git/local) | package registry | vendored install step | +| Smithy | Apache-2.0 | — | — | — | **Maven GAV** | **Maven JAR** (one ecosystem) | Maven resolver | +| CUE | Apache-2.0 | — | modules | module root | module paths | **OCI registries** | module resolution | +| Terraform | BUSL-1.1 (fork: OpenTofu, MPL-2.0) | — | — | — | **prefix-disambiguated string** | multi-scheme | lock file | +| Apicurio Registry | Apache-2.0 | — | — | — | group/id/version | — | **runtime**, not build | + +--- + +## Part 3 — What the evidence says about MetaObjects specifically + +Findings, not decisions. Each is a question the design must answer explicitly. + +1. **~~The ordered-list-of-roots primitive is settled prior art~~ — REVISED.** The Java port's + loader does take an ordered source list plus a URI grammar covering file, URL and classpath + resource, and four ports collapsed that to one directory, so propagation is still the job. But + the *ordering* half should not be propagated: per P8 no surveyed tool asks the author to + sequence anything, and per P1's correction this engine already ignores the declared order. + Propagate the multi-source capability and the `resource` kind; drop the sequence. + +2. **Collection identity is tooling config in every surveyed project — never part of the + schema language itself.** Buf, Cargo, Go, dbt, and graphql-config all keep "what is a + collection and where does it live" in a config file, entirely outside the modeled types. + That is direct evidence against introducing a `collection` node into the MetaObjects + metamodel, and in favor of the config layer — which also keeps `registry-conformance` and + ADR-0023 out of it. + +3. **Contextual discovery has one converged answer: nearest ancestor, plus an explicit + override, plus optionally a root config enumerating several collections.** graphql-config + ships all three shapes and documents when each applies. MetaObjects' CLI already has the + override (`--cwd` / a project-root positional); it is missing discovery. + +4. **Go's explicit-enumeration stance deserves weight over Cargo/Buf globbing**, because a + polyglot repo has more directories that merely *look* like collections, and silent + membership is the failure mode that is hardest to debug. + +5. **Local-path sources are the recommended monorepo mechanism in the two projects that + address monorepos head-on** (dbt explicitly; Terraform by giving local paths their own rule). + Both treat local paths as read-in-place, never installed. This is the shape the blocked + adopters need, and it is the cheapest thing in this document to ship. + +6. **The four-registry distribution plan is unprecedented among surveyed peers and needs an + argued decision.** The counter-evidence (CUE's explicit rejection, Buf's own registry, + Smithy's single ecosystem) is strong enough that "publish to all four" should be recorded as + an ADR with the reasoning, or reconsidered in favor of OCI — noting that MetaObjects' + existing four-registry lockstep release machinery is a genuine advantage none of these + projects had. + +7. **Remote sources need a fetch step and a pin. No surveyed tool reads a remote source inline + during a normal build.** A URL source resolved on every `meta gen` would be a novel shape, + and the novelty is a reproducibility cost, not a feature. + +8. **A database source is a runtime pattern, not a build-time one.** Splitting it out of the + build-time design entirely is consistent with every system surveyed. + +9. **The Buf workspace precedent does not transfer — verified by spike, not by reading.** Buf can + declare N modules and their consumers in one root file because **Buf owns its entire config + surface**. MetaObjects does not: the *build tool* owns it, and a consumer's generator wiring + already lives in `metaobjects.config.ts` or a `pom.xml`. A root config enumerating consumers + would have to duplicate or override those. Buf's *module-set* idea transfers; its *single root + config* does not. Recorded because the structural precondition, not the shape, is what decides + whether a borrowed pattern works. + +10. **Scoping should be package patterns, and MetaObjects already has the reference + implementation** (P9) — in the same port that has the multi-source list, and for the same + reason: it is the only port whose adopters hit these problems at scale. + +--- + +## Sources + +All URLs are public documentation, retrieved 2026-08-17. + +- protobuf / protoc — [compiler reference](https://protocolbuffers-protobuf-45.mintlify.app/tooling/protoc) · [include-path ordering idiom](https://jbrandhorst.com/post/go-protobuf-tips/) · [files and packages](https://buf.build/docs/reference/protobuf-files-and-packages/) +- Buf CLI — [modules and workspaces](https://buf.build/docs/cli/modules-workspaces/) · [v2 config migration](https://buf.build/docs/migration-guides/migrate-v2-config-files/) · [dependency management](https://buf.build/docs/bsr/module/dependency-management/) · [repo (Apache-2.0)](https://github.com/bufbuild/buf) +- Cargo — [workspaces](https://deepwiki.com/rust-lang/cargo/2.2-workspaces) +- Go modules — [reference](https://go.dev/ref/mod) +- graphql-config — [usage](https://the-guild.dev/graphql/config/docs/user/usage) · [multi-project config](https://the-guild.dev/graphql/codegen/docs/config-reference/multiproject-config) +- dbt — [packages](https://docs.getdbt.com/docs/build/packages) · [dbt_project.yml](https://docs.getdbt.com/reference/dbt_project.yml) · [project dependencies](https://docs.getdbt.com/docs/mesh/govern/project-dependencies) +- Smithy — [smithy-build.json](https://smithy.io/2.0/guides/smithy-build-json.html) · [Gradle plugins](https://smithy.io/2.0/guides/gradle-plugin/index.html) +- CUE — [modules reference](https://cuelang.org/docs/reference/modules/) · [custom module registry](https://cuelang.org/docs/tutorial/working-with-a-custom-module-registry/) · [modules v3 design proposal](https://github.com/cue-lang/proposal/blob/main/designs/modules.v3/2939-modules.md) +- Terraform — [module sources](https://developer.hashicorp.com/terraform/language/modules/sources) +- Apicurio Registry — [introduction](https://www.apicur.io/registry/docs/apicurio-registry/3.1.x/getting-started/assembly-intro-to-the-registry.html) · [artifact reference](https://www.apicur.io/registry/docs/apicurio-registry/3.3.x/getting-started/assembly-artifact-reference.html) diff --git a/fixtures/conformance/ERROR-CODES.json b/fixtures/conformance/ERROR-CODES.json index 8bd3a0dee..c5c5f22cd 100644 --- a/fixtures/conformance/ERROR-CODES.json +++ b/fixtures/conformance/ERROR-CODES.json @@ -17,6 +17,7 @@ "ERR_PROJECTION_INHERITED_SOURCE": "FR-024 (ADR-0028): a concrete object.projection inherits a source.* through extends instead of declaring its own. A projection's extends is shape lineage, not a shared-storage hierarchy: extends only ADDS members, so the child's extra fields have no provider in the parent's view, and two objects would claim one physical view with different declared exposures. Declare the source on the concrete projection; an abstract projection base carries shape only.", "ERR_INVALID_SUBTYPE_CHILD": "A child node type/subType is not permitted under its parent.", "ERR_CHILD_NOT_ALLOWED": "FR-033: a structural child (field/identity/source/validator/\u2026 \u2014 not an attr) is placed under a parent whose registered childRules do not admit it (the structural analogue of ERR_UNKNOWN_ATTR). Strict-load only; a no-op under wildcard childRules. The detail names the parent, the child (type.subType 'name'), and which placement was rejected.", + "ERR_COLLECTION_NOT_FOUND": "Phase-1 metadata-source-resolution: no metadata collection was discovered — no config declaring sources, and no default metaobjects/ directory.", "ERR_UNKNOWN_ATTR": "An attribute name is not declared on the node's type.", "ERR_MISSING_REQUIRED_ATTR": "A required attribute is absent from the node.", "ERR_BAD_ATTR_VALUE": "An attribute value fails its declared schema (type/range).", @@ -49,8 +50,11 @@ "ERR_OBJECT_FIELD_WITHOUT_OBJECT_REF": "ADR-0013: a field.object declares no @objectRef. A field.object models a typed nested value and REQUIRES @objectRef. For a genuinely open/untyped JSON map, use the physical escape hatch @dbColumnType: jsonb on a field.string instead of a bare object.", "ERR_UNRESOLVED_OBJECT_REF": "ADR-0042: a field.object / field.map @objectRef does not resolve to any object in the loaded tree (a dangling target). The ref resolves package-locally when bare (referrer's package, else root-level) and exactly when FQN \u2014 a bare cross-package ref no longer binds elsewhere. The error names same-short-name objects in other packages so the author can qualify it.", "ERR_RESERVED_ATTR": "An @-prefixed reserved structural keyword (e.g. @name, @isArray, @children) was used as an inline attribute.", + "ERR_SCOPE_PATTERN_INVALID": "Phase-1 metadata-source-resolution: a scope include/exclude package pattern is malformed (empty pattern or empty :: segment).", "ERR_SOURCE_NO_PRIMARY": "An object declares source nodes but none has role=primary.", "ERR_SOURCE_MULTIPLE_PRIMARY": "An object declares more than one source node with role=primary.", + "ERR_SOURCE_KIND_UNSUPPORTED": "Phase-1 metadata-source-resolution: a declared source kind (resource or package) is not supported by this toolchain.", + "ERR_SOURCE_UNRESOLVED": "Phase-1 metadata-source-resolution: a path source declared in .metaobjects/config.json does not exist on disk.", "ERR_PHYSICAL_NAME_KIND_MISMATCH": "FR-016 / ADR-0018: a source.rdb declares a kind-aware physical-name alias (@view/@materializedView/@proc/@function) that does not match its @kind. The legacy @table-for-non-table case warns rather than errors.", "ERR_PHYSICAL_NAME_MULTIPLE": "FR-016 / ADR-0018: a source.rdb declares two or more kind-aware physical-name aliases at once (e.g. both @table and @view). Exactly one is permitted.", "ERR_READONLY_ASSIGNED_PRIMARY": "FR-013: a field with @readOnly: true is the target of an identity.primary with @generation: \"assigned\". The application has no path to populate the identity value (no setter; not generated; not defaulted).", diff --git a/fixtures/scope-conformance/README.md b/fixtures/scope-conformance/README.md new file mode 100644 index 000000000..ec1d70f3f --- /dev/null +++ b/fixtures/scope-conformance/README.md @@ -0,0 +1,54 @@ +# scope-conformance + +Pins the pattern semantics of a consumer's `include`/`exclude` **scope** — the +filter over fully-qualified node names (`pkg::Sub::Name`) that decides which +metadata a source is authoritative for (phase 1 of metadata-source +resolution). `*` and `**` are easy to reinvent slightly differently per port; +this is the same failure mode that produced the cross-port `LIKE`/`ILIKE` +divergence fixed in 0.21.6. Every port's runner reads the single committed +`cases.json` — there is no per-port fixture and no ledger. + +## Shape + +``` +cases.json # { cases: [{ name, scope: {include?, exclude?}, expect: [{fqn, matches}] }] } +README.md +``` + +## Semantics + +- **Separator** is `::` (the package separator). A fully-qualified name is a + `::`-joined sequence of one or more segments. +- **`*`** inside a segment matches any run of characters, but **never crosses + a `::`** — it is scoped to a single segment. `acme::Order*` matches + `acme::OrderLine` but not `acme::deep::OrderLine`. +- **A segment that is exactly `**`** matches **one or more** whole segments. + `acme::**` matches `acme::Order` and `acme::a::b::Secret` but not the bare + `acme` (zero segments) — `**` never matches "nothing". `**` may also appear + mid-pattern (`acme::**::Order`), where it still requires at least one + segment between the fixed literals — `acme::Order` does **not** match + `acme::**::Order` (see `double-star-in-the-middle`). +- All other pattern characters (including regex metacharacters like `.`) are + **literal** — a pattern is never a general regex. +- **`include`** absent or empty means "everything is included". Otherwise a + name matches if **any** `include` pattern matches it (union). +- **`exclude`** is applied **after** `include` — a name excluded is excluded + regardless of which `include` pattern admitted it. `exclude` with no + `include` narrows the "everything" default. +- **Matching is case-sensitive** — a pattern and a name must agree in case + (`acme::Order` does not match `acme::order` or `ACME::Order`). + +## Behavioral contract + +Each port's runner reads `cases.json`, and for every case: compiles `scope` +with its native pattern compiler, then for every `expect` entry asserts +`matchesScope(fqn, compiledScope) === matches`. All ports assert the same +booleans — single-source, byte-identical expectations. + +## Reference implementation + +`server/typescript/packages/sdk/src/scope.ts` (`compileScope` / `matchesScope` +/ `compilePattern`) is the TypeScript reference this corpus was authored +against; other ports are free to implement the same semantics however is +idiomatic (e.g. a native regex engine, or a hand-rolled segment matcher) as +long as every case in this file passes. diff --git a/fixtures/scope-conformance/cases.json b/fixtures/scope-conformance/cases.json new file mode 100644 index 000000000..39ad0a81b --- /dev/null +++ b/fixtures/scope-conformance/cases.json @@ -0,0 +1,93 @@ +{ + "cases": [ + { + "name": "empty-scope-matches-everything", + "scope": {}, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "Order", "matches": true } + ] + }, + { + "name": "single-star-is-one-segment", + "scope": { "include": ["acme::*"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::commerce::Order", "matches": false }, + { "fqn": "other::Order", "matches": false } + ] + }, + { + "name": "double-star-is-one-or-more-segments", + "scope": { "include": ["acme::**"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::commerce::internal::Secret", "matches": true }, + { "fqn": "acme", "matches": false }, + { "fqn": "acmex::Order", "matches": false } + ] + }, + { + "name": "double-star-in-the-middle", + "scope": { "include": ["acme::**::Order"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::a::b::Order", "matches": true }, + { "fqn": "acme::Order", "matches": false }, + { "fqn": "acme::commerce::Invoice", "matches": false } + ] + }, + { + "name": "partial-star-never-crosses-separator", + "scope": { "include": ["acme::Order*"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::OrderLine", "matches": true }, + { "fqn": "acme::deep::OrderLine", "matches": false } + ] + }, + { + "name": "exclude-applied-after-include", + "scope": { "include": ["acme::**"], "exclude": ["acme::internal::**"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::internal::Secret", "matches": false } + ] + }, + { + "name": "exclude-alone-narrows-everything", + "scope": { "exclude": ["acme::internal::**"] }, + "expect": [ + { "fqn": "other::Thing", "matches": true }, + { "fqn": "acme::internal::Secret", "matches": false } + ] + }, + { + "name": "multiple-includes-are-a-union", + "scope": { "include": ["acme::commerce::**", "acme::common::**"] }, + "expect": [ + { "fqn": "acme::commerce::Order", "matches": true }, + { "fqn": "acme::common::BaseEntity", "matches": true }, + { "fqn": "acme::billing::Invoice", "matches": false } + ] + }, + { + "name": "regex-metacharacters-are-literal", + "scope": { "include": ["acme::Order.v2"] }, + "expect": [ + { "fqn": "acme::Order.v2", "matches": true }, + { "fqn": "acme::OrderXv2", "matches": false } + ] + }, + { + "name": "matching-is-case-sensitive", + "scope": { "include": ["acme::Order"] }, + "expect": [ + { "fqn": "acme::Order", "matches": true }, + { "fqn": "acme::order", "matches": false }, + { "fqn": "ACME::Order", "matches": false } + ] + } + ] +} diff --git a/server/csharp/MetaObjects/Errors.cs b/server/csharp/MetaObjects/Errors.cs index 2c6c8e6f5..94b9ce9da 100644 --- a/server/csharp/MetaObjects/Errors.cs +++ b/server/csharp/MetaObjects/Errors.cs @@ -128,6 +128,14 @@ public enum ErrorCode ERR_INVALID_TEMPLATE, ERR_SOURCE_NO_PRIMARY, ERR_SOURCE_MULTIPLE_PRIMARY, + // Phase-1 metadata-source-resolution: a path source declared in .metaobjects/config.json does not exist on disk. + ERR_SOURCE_UNRESOLVED, + // Phase-1 metadata-source-resolution: a declared source kind (resource or package) is not supported by this toolchain. + ERR_SOURCE_KIND_UNSUPPORTED, + // Phase-1 metadata-source-resolution: a scope include/exclude package pattern is malformed (empty pattern or empty :: segment). + ERR_SCOPE_PATTERN_INVALID, + // Phase-1 metadata-source-resolution: no metadata collection was discovered — no config declaring sources, and no default metaobjects/ directory. + ERR_COLLECTION_NOT_FOUND, // FR5c — multi-file overlay merge produced a conflicting attribute value: // two contributors set the same @attr to different non-empty values. ERR_MERGE_CONFLICT, diff --git a/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java b/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java index ef64e5921..4619a3ccf 100644 --- a/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java +++ b/server/java/metadata/src/main/java/com/metaobjects/ErrorCode.java @@ -283,6 +283,18 @@ public enum ErrorCode { /** An object declares more than one source node with role=primary. */ ERR_SOURCE_MULTIPLE_PRIMARY, + /** Phase-1 metadata-source-resolution: a {@code path} source declared in {@code .metaobjects/config.json} does not exist on disk. */ + ERR_SOURCE_UNRESOLVED, + + /** Phase-1 metadata-source-resolution: a declared source kind ({@code resource} or {@code package}) is not supported by this toolchain. */ + ERR_SOURCE_KIND_UNSUPPORTED, + + /** Phase-1 metadata-source-resolution: a {@code scope} include/exclude package pattern is malformed (empty pattern or empty {@code ::} segment). */ + ERR_SCOPE_PATTERN_INVALID, + + /** Phase-1 metadata-source-resolution: no metadata collection was discovered — no config declaring {@code sources}, and no default {@code metaobjects/} directory. */ + ERR_COLLECTION_NOT_FOUND, + /** * FR-016 / ADR-0018: a {@code source.rdb} declares a kind-aware physical-name * alias ({@code @view} / {@code @materializedView} / {@code @proc} / diff --git a/server/python/src/metaobjects/errors.py b/server/python/src/metaobjects/errors.py index d98915089..b2a8d11e8 100644 --- a/server/python/src/metaobjects/errors.py +++ b/server/python/src/metaobjects/errors.py @@ -104,6 +104,14 @@ class ErrorCode(str, Enum): # Source-v2 multi-source one-primary rule (ADR-0007). ERR_SOURCE_NO_PRIMARY = "ERR_SOURCE_NO_PRIMARY" ERR_SOURCE_MULTIPLE_PRIMARY = "ERR_SOURCE_MULTIPLE_PRIMARY" + # Phase-1 metadata-source-resolution — a path source declared in .metaobjects/config.json does not exist on disk. + ERR_SOURCE_UNRESOLVED = "ERR_SOURCE_UNRESOLVED" + # Phase-1 metadata-source-resolution — a declared source kind (resource or package) is not supported by this toolchain. + ERR_SOURCE_KIND_UNSUPPORTED = "ERR_SOURCE_KIND_UNSUPPORTED" + # Phase-1 metadata-source-resolution — a scope include/exclude package pattern is malformed (empty pattern or empty :: segment). + ERR_SCOPE_PATTERN_INVALID = "ERR_SCOPE_PATTERN_INVALID" + # Phase-1 metadata-source-resolution — no metadata collection was discovered: no config declaring sources, and no default metaobjects/ directory. + ERR_COLLECTION_NOT_FOUND = "ERR_COLLECTION_NOT_FOUND" # FR-016 / ADR-0018 — per-kind physical-name aliases on source.rdb. ERR_PHYSICAL_NAME_KIND_MISMATCH = "ERR_PHYSICAL_NAME_KIND_MISMATCH" ERR_PHYSICAL_NAME_MULTIPLE = "ERR_PHYSICAL_NAME_MULTIPLE" diff --git a/server/typescript/packages/cli/README.md b/server/typescript/packages/cli/README.md index 7f4f33d78..7a93fd44a 100644 --- a/server/typescript/packages/cli/README.md +++ b/server/typescript/packages/cli/README.md @@ -278,6 +278,84 @@ For D1 projects, the `migrate` block instead looks like: Precedence for `meta migrate`: CLI flag > env var (`DATABASE_URL` only) > `.metaobjects/config.json` > built-in default. +### Metadata sources (`sources`) + +`sources` is the single authority on **where metadata lives**. Every read command +(`gen`, `migrate`, `verify`, `docs`, `export`) resolves it. When the key is absent or +empty — which is what `meta init` scaffolds — it takes its default value, the +`metaobjects/` directory beside the config, so existing projects are unaffected. + +```jsonc +"sources": [ + { "path": "metaobjects" }, // a directory, relative to this config + { "path": "../model/src/main/resources/metadata" }, // a sibling module — read IN PLACE + { "path": "vendor/model/meta.catalog.json" } // a single file +] +``` + +- A relative `path` resolves against the directory holding this `.metaobjects/` + folder, never against the ambient cwd. A directory is walked recursively for + `.json` / `.yaml` / `.yml`, skipping `_pending/`. +- **`sources` is a set** — reordering it cannot change what resolves, and two entries + may overlap. +- A `path` that does not exist is an error (`ERR_SOURCE_UNRESOLVED`), never a silent + skip. +- `{ "resource": "…" }` and `{ "package": "…" }` parse but do not resolve yet + (`ERR_SOURCE_KIND_UNSUPPORTED`). The file is validated **strictly**, so a + misspelled key is a load error rather than a silently ignored extra. + +**Discovery.** The CLI walks **up** from the working directory (`--cwd` moves the +start) for the nearest `.metaobjects/config.json` — nearest wins — and stops after a +directory containing `.git`, so a checkout never adopts a parent checkout's config. + +### Output scope (`scope`) + +```jsonc +"scope": { + "include": ["acme::billing::**", "acme::common::*"], + "exclude": ["acme::billing::internal::**"] +} +``` + +Patterns match an object's fully-qualified name (`::`). `*` matches +within **one** segment and never crosses `::`; a segment that is exactly `**` matches +**one or more** whole segments; everything else is literal; absent/empty `include` +means everything; `exclude` applies after `include`; matching is case-sensitive. + +**The collection always loads in full — `scope` filters output, never input.** It +applies to `meta gen` and to `meta verify --codegen` (which regenerates under the +same scope, so a scoped `gen` is not reported as drift). `meta docs` and `meta export` +are deliberately **not** scoped — they inspect the loaded collection. `meta gen +` arguments intersect with `scope`; both must pass. + +The TypeScript-only per-generator `filter` function is unchanged and remains the +escape hatch for predicate-shaped filtering. + +### Per-command scope (`migrate.scope`) + +```jsonc +"migrate": { + "outDir": "./.metaobjects/migrations", + "scope": ["acme::billing::**"] +} +``` + +An include-only array of the same patterns, for a database this project **shares with +another owner**. Tables and views whose declaring object falls outside it are neither +created nor dropped — they leave the expected schema *and* are suppressed on the +actual side — and `meta migrate` prints how many it left alone. `meta verify --db` +reports them as out-of-scope instead of as drift. `meta migrate baseline` is +deliberately unscoped: a `--from-db` baseline has no metadata provenance to scope by. + +Declare a `migrate` block only in the project that holds `.metaobjects/migrations/` +and the schema snapshot — that project owns the schema. Run `meta migrate` from that +directory (or point `--cwd` at it): `migrate.scope` comes from the discovered config, +but the rest of the block is read from `.metaobjects/config.json` in the directory the +command runs in. + +Full adopter guide, including vendoring for airgapped builds and a worked polyglot +example: [docs/features/metadata-sources.md](../../../../docs/features/metadata-sources.md). + ## Metadata format See `.metaobjects/AGENTS.md` (scaffolded by `meta init`) for the metaobjects metamodel rules, attribute conventions, and worked examples. Deeper references: diff --git a/server/typescript/packages/cli/src/commands/docs.ts b/server/typescript/packages/cli/src/commands/docs.ts index 84451ba9b..84ebf67b6 100644 --- a/server/typescript/packages/cli/src/commands/docs.ts +++ b/server/typescript/packages/cli/src/commands/docs.ts @@ -15,7 +15,7 @@ import { resolve as resolvePath, basename } from "node:path"; import { mkdir, writeFile } from "node:fs/promises"; import { log } from "../lib/log.js"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; -import { loadMemory, DEFAULT_METADATA_DIR } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection, resolveConfigDir, type Collection } from "@metaobjectsdev/sdk"; import { existsSync } from "node:fs"; import { join } from "node:path"; import { @@ -206,16 +206,41 @@ export async function docsCommand(args: string[], cwd: string): Promise // `--scaffold-site`: copy the docs-site templates + assets into codegen/docs-site/ // so the consumer owns them (ADR-0034 scaffold-and-own). Scaffold and return — - // it does not also generate. + // it does not also generate. `resolveConfigDir` rather than `resolveCollection`: + // scaffolding needs no metadata, but it must write where `emitSite` will READ + // (below, under the resolved project root), and that is the same walk. if (flags.scaffoldSite) { - return scaffoldSiteCommand(metaRoot); + return scaffoldSiteCommand(await resolveConfigDir(metaRoot)); + } + + // Discovery and load are two separate failure modes, kept in separate try + // blocks deliberately — same reasoning as `meta gen` (gen.ts): a broad + // catch around both would swallow a genuine ParseError as "no metaobjects/ + // found", masking the real failure. + // + // Discovery runs BEFORE the config read, deliberately, and `meta gen` calls + // out the same ordering as the thing it fixed: the project root is whichever + // directory `resolveCollection` decided the metadata belongs to, so + // everything project-relative — `metaobjects.config.ts` and its providers, + // the `docs.outDir` it names, the adopter `templates/` overrides, the owned + // `codegen/docs-site/` theme — has to come from that same directory. Reading + // the config from the ambient `` argument while the metadata came + // from an ancestor renders the ancestor's model with the subdirectory's + // (absent) providers. For a run at the project root the two are the same path. + let collection: Awaited>; + try { + collection = await resolveCollection(metaRoot); + } catch (err) { + log.error(`docs: ${(err as Error).message}`); + return 2; } // The project root used to resolve adopter `templates/` overrides; the - // framework defaults sit underneath via projectProvider's chain. + // framework defaults sit underneath via projectProvider's chain. `--templates` + // is the one explicit override. const projectRoot = flags.templates !== undefined ? resolvePath(cwd, flags.templates) - : metaRoot; + : collection.configDir; // Best-effort load of metaobjects.config.ts to pick up consumer-supplied // providers (e.g. a project's custom field/object subtypes). Unlike `gen`, @@ -228,9 +253,10 @@ export async function docsCommand(args: string[], cwd: string): Promise // hasConfig gates the api surface: api docs describe the GENERATED REST // surface, which only exists when there is a (loadable) gen config. A config // that EXISTS but fails to load degrades to model-only with a warning. - const hasConfig = existsSync(join(metaRoot, "metaobjects.config.ts")); - // The config lives alongside metaobjects/ at the metadata root (metaRoot); - // projectRoot only diverges when --templates overrides the template lookup. + const hasConfig = existsSync(join(collection.configDir, "metaobjects.config.ts")); + // The config lives beside the `.metaobjects/` that declared this collection — + // `collection.configDir`, never ambient cwd. `projectRoot` only diverges from + // it when --templates overrides the template lookup. // Only attempt the load when the file is actually present: absence is the // expected config-less case (stay silent), but a config that EXISTS yet fails // to load is surfaced as a warning rather than silently degrading to @@ -238,7 +264,7 @@ export async function docsCommand(args: string[], cwd: string): Promise // cryptic unknown-subtype error instead of the real config error. if (hasConfig) { try { - loadedConfig = await loadMetaobjectsConfig(metaRoot); + loadedConfig = await loadMetaobjectsConfig(collection.configDir); configProviders = loadedConfig.providers; } catch (err) { log.warn( @@ -269,7 +295,7 @@ export async function docsCommand(args: string[], cwd: string): Promise cliOverrides, loadedConfig?.outputLayout ?? "flat", ); - const outDir = resolvePath(metaRoot, docsCfg.outDir); + const outDir = resolvePath(collection.configDir, docsCfg.outDir); // SITE surface has its OWN model loader (docs-site's loadModel — NOT the sdk // loadMemory below) and needs no gen config. When the site is the ONLY @@ -277,23 +303,19 @@ export async function docsCommand(args: string[], cwd: string): Promise // WITHOUT building the markdown GenContext — decoupled and one fewer failure // surface. Combined with --model/--api it is emitted after them (below). if (flags.site && docsCfg.surfaces.length === 0) { - return emitSite(metaRoot, outDir, configProviders, promptsDir); + return emitSite(collection, projectRoot, outDir, configProviders, promptsDir); } // Load metadata standalone — same loader path as migrate/gen. Threads any // consumer providers from the config so custom types resolve. let root; try { - root = await loadMemory(metaRoot, { + root = await loadMemory(collection.configDir, { + files: collection.files, ...(configProviders !== undefined ? { providers: configProviders } : {}), }); } catch (err) { - const msg = (err as Error).message; - if (!existsSync(join(metaRoot, DEFAULT_METADATA_DIR))) { - log.error(`docs: no metaobjects/ found in ${metaRoot}; run 'meta init' to scaffold`); - } else { - log.error(`docs: failed to load metadata: ${msg}`); - } + log.error(`docs: failed to load metadata: ${(err as Error).message}`); return 2; } @@ -444,7 +466,7 @@ export async function docsCommand(args: string[], cwd: string): Promise // SITE surface (additive) — emit after the markdown surfaces so both coexist. if (flags.site) { - const siteRc = await emitSite(metaRoot, outDir, configProviders, promptsDir); + const siteRc = await emitSite(collection, projectRoot, outDir, configProviders, promptsDir); if (siteRc !== 0) return siteRc; } @@ -471,9 +493,9 @@ export async function docsCommand(args: string[], cwd: string): Promise * into `/codegen/docs-site/{templates,assets}`, writing each file ONLY if * absent so a re-run never clobbers a hand-edited file. */ -async function scaffoldSiteCommand(metaRoot: string): Promise { - const tplDir = join(metaRoot, "codegen/docs-site/templates"); - const astDir = join(metaRoot, "codegen/docs-site/assets"); +async function scaffoldSiteCommand(projectRoot: string): Promise { + const tplDir = join(projectRoot, "codegen/docs-site/templates"); + const astDir = join(projectRoot, "codegen/docs-site/assets"); const created: string[] = []; const preserved: string[] = []; try { @@ -499,53 +521,75 @@ async function scaffoldSiteCommand(metaRoot: string): Promise { } log.info( `meta docs --scaffold-site — ${created.length} created, ${preserved.length} preserved ` + - `→ ${join(metaRoot, "codegen/docs-site")} (edit these to own your theme)`, + `→ ${join(projectRoot, "codegen/docs-site")} (edit these to own your theme)`, ); return 0; } /** * Emit the browsable HTML documentation site via `@metaobjectsdev/docs-site`. - * The site loads the model with its OWN loader from the metadata source dir - * (`/metaobjects`), so this is independent of the sdk loadMemory path - * used for the markdown surfaces. Writes under `/site` so it can coexist - * with the markdown output. Scaffold-and-own: when the consumer has copied - * templates/assets into `/codegen/docs-site/` (via `--scaffold-site`), - * those win over the bundled defaults. + * The site loads the model with its OWN loader from the collection's declared + * source ROOTS (whole directories, one page group each) rather than from the + * per-file list the sdk `loadMemory` path takes, so this is independent of the + * markdown surfaces. Writes under `/site` so it can coexist with the markdown + * output. Scaffold-and-own: when the consumer has copied templates/assets + * into `/codegen/docs-site/` (via `--scaffold-site`), those win + * over the bundled defaults. + * + * Takes the ALREADY-RESOLVED collection: `docsCommand` resolved it to read the + * config from the right directory, and resolving a second time here made the + * combined `--model --site` path do the whole discovery-and-config walk twice. */ async function emitSite( - metaRoot: string, + collection: Collection, + projectRoot: string, outDir: string, configProviders?: readonly MetaDataTypeProvider[], promptsDir?: string, ): Promise { const siteOutDir = resolvePath(outDir, "site"); - // metaobjects/ is REQUIRED (the site loads the model from it) and always first. - // Prompt `.mustache` source is additionally searched in the conventional - // /templates/ and any explicit --prompts dir (for a project whose templates - // live elsewhere, e.g. data/templates/) — else the site can't show the prompt TEXT - // and prints a "source missing" note. Only existing dirs are added, and dirs are - // deduped by BASENAME (the site keys source groups by basename, and rejects a dup). - const sourceDirs = [join(metaRoot, DEFAULT_METADATA_DIR)]; - const seenBasenames = new Set([basename(join(metaRoot, DEFAULT_METADATA_DIR))]); + // The resolved metadata source dir(s) are REQUIRED (the site loads the + // model from them) and always first. Prompt `.mustache` source is + // additionally searched in the conventional /templates/ and any + // explicit --prompts dir (for a project whose templates live elsewhere, + // e.g. data/templates/) — else the site can't show the prompt TEXT and + // prints a "source missing" note. Only existing dirs are added. + // + // Deduped by resolved PATH, not by basename. Two DIFFERENT directories that + // happen to share a basename are a legitimate multi-source project (`metaobjects` + // plus `../shared-model/metaobjects`); `loadModel` disambiguates their site + // group names, so refusing the pair here — which a basename key did, by + // dropping the second — would break the feature this branch exists to ship. + // The same directory named twice is the real hazard: it would be symlinked + // and loaded twice. + // The DECLARED source roots, not directories re-derived from the resolved + // files: a declared source directory holding no metadata yet would otherwise + // vanish from the site's group list entirely, and `sourceDirs` could come back + // empty where the pre-branch code always passed `/metaobjects`. + const sourceDirs = [...collection.sourceRoots]; + const seenDirs = new Set(sourceDirs); if (promptsDir !== undefined && !existsSync(promptsDir)) { log.warn(`docs: --prompts dir does not exist: ${promptsDir}`); } - for (const d of [join(metaRoot, "templates"), ...(promptsDir !== undefined ? [promptsDir] : [])]) { - if (existsSync(d) && !seenBasenames.has(basename(d))) { - sourceDirs.push(d); - seenBasenames.add(basename(d)); + for (const d of [join(projectRoot, "templates"), ...(promptsDir !== undefined ? [promptsDir] : [])]) { + const abs = resolvePath(d); + if (existsSync(abs) && !seenDirs.has(abs)) { + sourceDirs.push(abs); + seenDirs.add(abs); } } // Scaffold-and-own: when the consumer has copied templates/assets into // codegen/docs-site/ (via --scaffold-site), use those; else the bundled defaults. - const ownedTemplates = join(metaRoot, "codegen/docs-site/templates"); - const ownedAssets = join(metaRoot, "codegen/docs-site/assets"); + // Keyed on `configDir`, NOT `projectRoot`: `--templates` redirects the adopter + // RENDER template chain (the `templates/` above), and letting it also move the + // docs-site theme would read it from somewhere `--scaffold-site` never writes. + const ownedTemplates = join(collection.configDir, "codegen/docs-site/templates"); + const ownedAssets = join(collection.configDir, "codegen/docs-site/assets"); try { const r = await generateSite({ sourceDirs, outDir: siteOutDir, - title: basename(metaRoot) || "Metadata", + title: basename(collection.configDir) || "Metadata", stamp: new Date().toISOString().slice(0, 10), commit: "", core: { n: 15 }, diff --git a/server/typescript/packages/cli/src/commands/export.ts b/server/typescript/packages/cli/src/commands/export.ts index 46109099e..e12d65f11 100644 --- a/server/typescript/packages/cli/src/commands/export.ts +++ b/server/typescript/packages/cli/src/commands/export.ts @@ -1,10 +1,10 @@ -import { resolve, join } from "node:path"; +import { resolve } from "node:path"; import { writeFile } from "node:fs/promises"; import { parseExportArgs } from "../lib/args.js"; import { log } from "../lib/log.js"; -import { loadAndExportJson } from "@metaobjectsdev/metadata/core"; -import { TypeRegistry, registerCoreTypes } from "@metaobjectsdev/metadata"; -import { DEFAULT_METADATA_DIR, registerForgeTypes } from "@metaobjectsdev/sdk"; +import { FileSource } from "@metaobjectsdev/metadata/core"; +import { TypeRegistry, registerCoreTypes, MetaDataLoader, canonicalSerialize } from "@metaobjectsdev/metadata"; +import { registerForgeTypes, resolveCollection } from "@metaobjectsdev/sdk"; export async function exportCommand(args: string[], cwd: string): Promise { let flags; @@ -16,7 +16,6 @@ export async function exportCommand(args: string[], cwd: string): Promise new FileSource(f)), + ); + const result = { + json: canonicalSerialize(loadResult.root), + errors: loadResult.errors, + warnings: loadResult.warnings.map((w) => w.message), + }; for (const w of result.warnings) { log.warn(w); diff --git a/server/typescript/packages/cli/src/commands/gen.ts b/server/typescript/packages/cli/src/commands/gen.ts index f260b426e..6f7ae4238 100644 --- a/server/typescript/packages/cli/src/commands/gen.ts +++ b/server/typescript/packages/cli/src/commands/gen.ts @@ -1,5 +1,4 @@ -import { relative, join } from "node:path"; -import { existsSync } from "node:fs"; +import { relative } from "node:path"; import { parseGenArgs } from "../lib/args.js"; import { resolveGenConfig } from "../lib/config.js"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; @@ -9,7 +8,7 @@ import type { OutputFormat } from "../lib/format.js"; import { log } from "../lib/log.js"; import { warnIfAgentContextStale } from "../lib/agent-context-staleness.js"; import { scanSourceForAntiPatterns } from "../lib/anti-patterns.js"; -import { loadMemory, DEFAULT_METADATA_DIR } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; import { runGen, listGenerators } from "@metaobjectsdev/codegen-ts"; import type { WriteStatus } from "@metaobjectsdev/codegen-ts"; @@ -36,12 +35,41 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat return listGeneratorsCommand(); } - // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. - warnIfAgentContextStale(cwd); - - const projectRoot = cwd; const cliConfig = resolveGenConfig(flags); + // Discovery and load are two separate failure modes, kept in separate try + // blocks deliberately: a broad catch around both previously swallowed + // genuine ParseErrors (e.g. `origin.@via "X.y" ...: no such relationship + // "y" on X`) as "no metaobjects/ found", masking the real failure. + // `resolveCollection` raises `ERR_COLLECTION_NOT_FOUND` with its own + // message when nothing is discovered and no default directory exists. + // + // Discovery runs BEFORE the config read, deliberately: the project root is + // whichever directory `resolveCollection` decided the metadata belongs to, + // and everything project-relative — `metaobjects.config.ts`, the `outDir` + // its generators name, `.metaobjects/.gen-state/` — has to come from that + // same directory. Reading the config from ambient cwd while the metadata + // came from an ancestor is the config-half of the very divergence this + // design exists to remove (design §4.6.1: "Per-port generator config is then + // read from that same directory"). For a run from the project root the two + // are the same path, which is the only invocation that worked before. + let collection; + try { + collection = await resolveCollection(cwd); + } catch (err) { + log.error((err as Error).message); + return 2; + } + const projectRoot = collection.configDir; + + // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. + // Rooted at `projectRoot`, not ambient cwd — the scaffolded agent context sits + // with the project that declares the metadata, so a run from a subdirectory + // would find no manifest there and silently skip the nudge. `meta verify` makes + // the same call for the same reason; the two commands describe this and the + // anti-pattern scan below as one advisory pass, so they must scan one tree. + warnIfAgentContextStale(projectRoot); + let forgeConfig; try { forgeConfig = await loadMetaobjectsConfig(projectRoot); @@ -52,22 +80,12 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat let metadata; try { - metadata = await loadMemory(projectRoot, { + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(forgeConfig.providers !== undefined ? { providers: forgeConfig.providers } : {}), }); } catch (err) { - const msg = (err as Error).message; - // Only emit the scaffold hint for the ACTUAL missing-metadata-dir - // condition — checked explicitly here. A broad substring match on - // "no such" / "cannot read" wrongly swallowed genuine ParseErrors (e.g. - // `origin.@via "X.y" ...: no such relationship "y" on X`) as "no - // metaobjects/ found", masking the real failure. Real parse/validation - // errors propagate with their actual message. - if (!existsSync(join(projectRoot, DEFAULT_METADATA_DIR))) { - log.error(`no metaobjects/ found in ${projectRoot}; run 'meta init' to scaffold`); - } else { - log.error(`failed to load metadata: ${msg}`); - } + log.error(`failed to load metadata: ${(err as Error).message}`); return 2; } @@ -81,6 +99,11 @@ export async function genCommand(args: string[], cwd: string, fmt: OutputFormat // --dry-run must actually preview. This was previously passed only to the // display object below, so a "preview" run wrote every file. dryRun: cliConfig.dryRun, + // Collection-level `scope` (Task 12b) — the output filter over + // GENERATED entities, never over what the collection loads. Always + // passed: an unconfigured project's predicate admits everything, so this + // is a no-op for the common case, not a behavior change. + scope: collection.inScope, ...(cliConfig.entities.length > 0 ? { entityFilter: cliConfig.entities } : {}), }); } catch (err) { diff --git a/server/typescript/packages/cli/src/commands/init.ts b/server/typescript/packages/cli/src/commands/init.ts index 00c9f4f79..73f655a19 100644 --- a/server/typescript/packages/cli/src/commands/init.ts +++ b/server/typescript/packages/cli/src/commands/init.ts @@ -192,7 +192,7 @@ function warnIfMonorepoSubdir(opts: InitOptions, result: InitResult): void { * string[]s (or nothing) straight through is safe — no need to special-case an empty * or absent prior. */ -function stackForAgentContext(opts: InitOptions, prior: Manifest | undefined): Stack { +async function stackForAgentContext(opts: InitOptions, prior: Manifest | undefined): Promise { const hasOverride = (opts.servers?.length ?? 0) > 0 || (opts.clients?.length ?? 0) > 0; const overrides = hasOverride ? { servers: opts.servers ?? [], clients: opts.clients ?? [] } @@ -203,7 +203,7 @@ function stackForAgentContext(opts: InitOptions, prior: Manifest | undefined): S async function writeAgentContext(opts: InitOptions, result: InitResult): Promise { warnIfMonorepoSubdir(opts, result); const prior = await readManifest(opts.cwd); - const stack = stackForAgentContext(opts, prior); + const stack = await stackForAgentContext(opts, prior); let assembled = assemble({ contentRoot: resolveAgentContextRoot(), stack }); if (opts.noSkills) assembled = assembled.filter((f) => !f.path.startsWith(".claude/skills/")); diff --git a/server/typescript/packages/cli/src/commands/migrate.ts b/server/typescript/packages/cli/src/commands/migrate.ts index 41c0a9dad..3d6041145 100644 --- a/server/typescript/packages/cli/src/commands/migrate.ts +++ b/server/typescript/packages/cli/src/commands/migrate.ts @@ -1,5 +1,6 @@ import { resolve as resolvePath } from "node:path"; import { mkdir } from "node:fs/promises"; +import { existsSync } from "node:fs"; import { spawn } from "node:child_process"; import { parseMigrateArgs } from "../lib/args.js"; import { resolveMigrateConfig, MIGRATE_DEFAULT_OUT_DIR } from "../lib/config.js"; @@ -10,13 +11,17 @@ import type { OutputFormat } from "../lib/format.js"; import { toonEncode } from "../lib/format.js"; import { buildKyselyFromUrl, redactUrl } from "../lib/kysely.js"; import { log } from "../lib/log.js"; -import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection, resolveConfigDir, type Collection } from "@metaobjectsdev/sdk"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; +import { migrateScopeMismatch, outOfScopeNote } from "../lib/migrate-scope.js"; import { - buildExpectedSchema, + buildExpectedSchemaWithProvenance, + scopeExpectedSchema, + scopedDiffInputs, introspect, diff, collectUnmanagedNames, + carryForwardOutOfScope, emit, writeMigration, baselineFromMetadata, @@ -41,6 +46,7 @@ import { type D1Binding, type EmitResult, type D1Runner, + type SchemaProvenance, } from "@metaobjectsdev/migrate-ts"; import { buildWranglerExecuteArgs, @@ -122,6 +128,74 @@ function resolveFormatOutDir(config: ResolvedMigrateConfig, metaRoot: string): s return resolvePath(metaRoot, config.outDir); } +/** + * D1's OWN directory convention — `--out-dir` > `wrangler.toml`'s + * `migrations_dir` > `"migrations"` — kept apart from `resolveFormatOutDir` + * because the middle term is not knowable until a wrangler binding has been + * resolved (`runD1Migrate` step 1), while every other dialect can answer + * immediately from `config` alone. + */ +function resolveD1OutDir( + config: ResolvedMigrateConfig, + metaRoot: string, + migrationsDirHint: string | undefined, +): string { + const isDefaultOutDir = config.outDir === MIGRATE_DEFAULT_OUT_DIR; + return resolvePath(metaRoot, isDefaultOutDir ? (migrationsDirHint ?? "migrations") : config.outDir); +} + +/** + * Say so when the migrations directory this run will use is NOT the one sitting + * in the working directory. + * + * `metaRoot` is now the discovered project root rather than ambient cwd, and the + * migrations directory follows it. That is the right call — the ledger belongs + * with the config that declares it — but it is a behaviour change for the two + * subcommands that load no metadata at all: `apply-pending` and `--rollback` + * used cwd unconditionally, so a subdirectory holding `.metaobjects/migrations` + * under a project root that also has one now replays the ROOT's ledger. + * Replaying somebody else's migration history silently is the worst outcome + * available here, so it is announced. + * + * Conditioned on the local directory EXISTING, so the ordinary case — a run from + * anywhere inside a project with one ledger at its root — says nothing. + * `--out-dir` (and a `migrate.outDir` in the config) is honoured: the caller + * must pass the directory THIS run will actually WRITE to, never a default + * that was overridden — a deliberate redirection is compared, not a stale + * guess. That answer comes from **two** call sites, because there are two + * directory conventions: every dialect but d1 resolves via + * `resolveFormatOutDir` before the format/dialect dispatch below; d1 has its + * own convention (`resolveD1OutDir`, wrangler.toml's `migrations_dir`) that is + * unknowable until its binding resolves, so it calls this again for itself + * from inside `runD1Migrate`, once that binding is in hand. + */ +function warnIfLedgerRelocated(cwd: string, resolvedOutDir: string): void { + const local = resolvePath(cwd, MIGRATE_DEFAULT_OUT_DIR); + if (resolvedOutDir === local || !existsSync(local)) return; + log.warn( + `migrate: using the migrations directory ${resolvedOutDir}, not the ${local} ` + + `in this working directory — the ledger belongs to the project root that declares it. ` + + `Pass --out-dir ${local} to use the local one.`, + ); +} + +/** + * Report what a declared `migrate.scope` left out (wording: `outOfScopeNote`). + * + * STDOUT in text format, STDERR otherwise. `--format json` / `--format toon` put a + * single machine-readable document on stdout, and a prose line ahead of it breaks + * `| jq` outright — the same split `emitStructuredError` makes just below. + * Routed to stderr rather than dropped, because the non-TTY default format is toon + * (`resolveFormat`): suppressing it outright would silence the note for every + * piped and CI run, which is most of them. + */ +function logOutOfScope(names: readonly string[], fmt: OutputFormat): void { + if (names.length === 0) return; + const msg = outOfScopeNote("migrate", names); + if (fmt === "text") log.info(msg); + else log.warn(msg); +} + function emitStructuredError(error: string, hint: string, fmt: OutputFormat): void { const payload = { error, hint }; if (fmt === "json") { @@ -132,6 +206,27 @@ function emitStructuredError(error: string, hint: string, fmt: OutputFormat): vo // text format: errors go to stderr via log.error() — the caller handles that path } +/** + * The refusal for a `migrate.scope` that matches nothing, as all three of migrate's + * pipelines (online, offline, D1) issue it. + * + * Returns the exit code to return, or `undefined` when there is nothing to refuse. + * Three byte-identical copies of the report-and-exit differed only in a local + * variable name; the hint string and the exit code are one decision, recorded once + * — a configuration error, so exit 2. + */ +function refuseScopeMismatch( + collection: Collection, + provenance: () => SchemaProvenance, + fmt: OutputFormat, +): number | undefined { + const mismatch = migrateScopeMismatch(collection, provenance); + if (mismatch === undefined) return undefined; + log.error(`migrate: ${mismatch}`); + emitStructuredError(`migrate: ${mismatch}`, "fix or remove migrate.scope in .metaobjects/config.json", fmt); + return 2; +} + /** * Sentinel thrown by sub-functions that have already emitted a structured error * via emitStructuredError(). The top-level catch in migrateCommand re-throws @@ -257,8 +352,45 @@ export async function migrateCommand( return 2; } - const metaRoot = cwd; + // The project root is the directory whose `.metaobjects/config.json` governs + // this run — the same directory `resolveCollection` resolves the metadata + // from, found the same way (design §4.6.1: "Per-port generator config is then + // read from that same directory"). Everything below is relative to it: the + // `.metaobjects/config.json` operational block, `metaobjects.config.ts`'s + // `columnNamingStrategy`, the migrations `outDir`, `wrangler.toml` discovery. + // + // Read from ambient cwd instead, as this did, they DIVERGE the moment the two + // differ: run `meta migrate` from a subdirectory of a project whose root + // declares `columnNamingStrategy: "literal"` and the metadata resolves from + // the ancestor while the strategy silently defaults to snake_case — emitting a + // migration that renames every column. Newly reachable, too: before metadata + // sources were resolvable that invocation just failed with "no metaobjects/ + // found". + // + // `resolveConfigDir` rather than `resolveCollection` deliberately: this must + // not require metadata to EXIST. `migrate apply-pending` and `--rollback` + // replay committed SQL and load no metadata at all, and making them fail on a + // project with no model would be a regression. It is the SAME walk + // `resolveCollection` runs (one exported definition in the sdk's + // `discovery.ts`, not two that agree by construction), so the directory this + // resolves and the directory the metadata comes from cannot diverge. + const metaRoot = await resolveConfigDir(cwd); const config = await resolveMigrateConfig(flags, metaRoot); + // `resolveFormatOutDir`, not `resolvePath(metaRoot, config.outDir)`: under + // `--migration-format flyway` with a default outDir the run writes to + // Flyway's conventional location instead, so the unredirected path names a + // directory this invocation will never touch. + // + // Skipped for a plain `--dialect d1` run (format !== flyway): d1 resolves + // its OWN directory from wrangler.toml, unknowable until its binding + // resolves, so it issues this warning for itself from inside + // `runD1Migrate` instead — `resolveFormatOutDir` here would name the + // Kysely-path default, a directory that run never writes to. A d1 + + // `--migration-format flyway` run is refused just below before either + // directory is ever touched, so THAT combination still wants this one. + if (config.dialect !== "d1" || config.format === "flyway") { + warnIfLedgerRelocated(cwd, resolveFormatOutDir(config, metaRoot)); + } try { // #192 — Flyway owns apply + history (flyway_schema_history). We generate the @@ -341,7 +473,7 @@ export async function migrateCommand( ); return 2; } - return await runD1Migrate(config, metaRoot, wranglerRunner ?? defaultWranglerRunner, fmt); + return await runD1Migrate(config, metaRoot, cwd, wranglerRunner ?? defaultWranglerRunner, fmt); } // `migrate baseline` — seed the committed reference snapshot, emit no migration. @@ -389,18 +521,27 @@ export async function migrateCommand( postgresConfigProviders = undefined; } + // Discovery and load are two separate failure modes, kept in separate try blocks + // (the `meta gen` pattern): a broad catch around both reports a genuine ParseError + // as "no metadata found", masking the real failure. `resolveCollection` raises + // ERR_COLLECTION_NOT_FOUND with its own message — the same exit 2 the hand-rolled + // ENOENT sniff used to produce. + let collection; + try { + collection = await resolveCollection(metaRoot); + } catch (err) { + log.error((err as Error).message); + return 2; + } + let metadata; try { - metadata = await loadMemory(metaRoot, { + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(postgresConfigProviders !== undefined ? { providers: postgresConfigProviders } : {}), }); } catch (err) { - const msg = (err as Error).message; - if (msg.includes("ENOENT") || msg.includes("no such") || msg.includes("cannot read")) { - log.error(`no metaobjects/ found in ${metaRoot}; run 'meta init' to scaffold`); - } else { - log.error(`failed to load metadata: ${msg}`); - } + log.error(`failed to load metadata: ${(err as Error).message}`); return 2; } @@ -435,11 +576,20 @@ export async function migrateCommand( // view DDL (create/drop/replace + dependency-recreate) and emit() renders it — // there is no separate view-migration emitter. const expectedViews = buildProjectionViews(metadata, { dialect: kysely.dialect, columnNamingStrategy }); - const expected = buildExpectedSchema(metadata, { + const built = buildExpectedSchemaWithProvenance(metadata, { dialect: kysely.dialect, columnNamingStrategy, views: expectedViews, }); + const scopeRc = refuseScopeMismatch(collection, () => built.provenance, fmt); + if (scopeRc !== undefined) return scopeRc; + // Per-command scope: objects outside `migrate.scope` are another owner's. They + // leave the expected schema here and are suppressed on the actual side below — + // dropping them from `expected` ALONE would propose DROP TABLE for every one of + // them that exists in the database. + const scoped = scopeExpectedSchema(built, collection.inMigrateScope); + const expected = scoped.snapshot; + logOutOfScope(scoped.outOfScope, fmt); let actual; try { actual = await introspect(kysely.db, kysely.dialect); @@ -455,7 +605,12 @@ export async function migrateCommand( let diffResult; try { diffResult = await diff({ - expected, + // The three scoped-diff obligations as one value (migrate-ts's scope.ts + // header has the mechanism): the narrowed expected side, `unmanagedNames` + // merging #208 §7's declared-@unmanaged set with the out-of-scope names so + // neither is created or dropped, and the schema scope pinned to the + // UNSCOPED model so narrowing can never widen the run. + ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, dialect: kysely.dialect, allow: tokensToAllowOptions(config.allow), @@ -463,9 +618,6 @@ export async function migrateCommand( // has no expressible migration; refuse loudly instead of emitting SQL that drops // the constraint and breaks referencing FKs at apply. refusePrimaryKeyChange: true, - // #208 §7 — declared-@unmanaged objects are external: exclude them from the - // actual side so migrate proposes neither create nor drop for them. - unmanagedNames: collectUnmanagedNames(metadata), onAmbiguous: async (a) => { collectedAmbiguous.push(a); return onAmbiguousResolution; @@ -610,9 +762,25 @@ export async function migrateCommand( // native ALTER vs recreate-and-copy on older SQLite. if (!config.dryRun && exitCode === 0 && !applyFailed && writtenPaths.length > 0) { try { + // The COMMITTED snapshot keeps what `migrate.scope` excluded: writing the + // narrowed schema would delete every out-of-scope entry, so a later widening + // would propose CREATE TABLE for a table that exists and fail at apply. The + // out-of-scope entries come from `actual` — they are in the database, which + // is the same thing `baseline --from-db` records. Identical object, and so a + // byte-identical snapshot, for an unscoped run. + // + // The trade-off, stated so it is not rediscovered: those carried entries are + // INTROSPECTED descriptors, not metadata-built ones, so they can differ + // cosmetically from what this model would have emitted for the same table + // (column order, a default's rendered form). Removing the scope later can + // therefore produce one round of alter churn. That is strictly better than + // the alternative it replaced — a `CREATE TABLE` for a table that exists, + // which fails at apply — and it is the same mixed-provenance snapshot + // `baseline --from-db` writes for every table it adopts. + const committed = carryForwardOutOfScope(expected, actual, scoped.outOfScope); await writeSnapshot( snapshotPath(resolvePath(metaRoot, config.outDir), kysely.dialect), - actual.meta !== undefined ? { ...expected, meta: actual.meta } : expected, + actual.meta !== undefined ? { ...committed, meta: actual.meta } : committed, ); } catch (err) { // The migration itself is written (and possibly applied) — report the @@ -745,8 +913,19 @@ export async function runBaseline( } catch { // config absent — no custom providers, default snake_case } + // `baseline` records a STARTING POINT, so it is deliberately NOT scoped: the + // `--from-db` arm captures whatever the database holds (there is no provenance + // for an introspected table), and an offline baseline that recorded less would + // disagree with it. An out-of-scope table sitting in the snapshot is harmless: + // every later run suppresses it on both sides of the diff, and an accepted + // scoped run carries it FORWARD (`carryForwardOutOfScope`) rather than dropping + // it — which is what makes that true. Committing the narrowed schema instead + // would delete the entry, and removing the scope later would then propose + // CREATE TABLE for a table that exists. try { - metadata = await loadMemory(metaRoot, { + const collection = await resolveCollection(metaRoot); + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(baselineConfigProviders !== undefined ? { providers: baselineConfigProviders } : {}), }); } catch (err) { @@ -880,8 +1059,11 @@ export async function runOfflineGenerate( } let metadata; + let collection; try { - metadata = await loadMemory(metaRoot, { + collection = await resolveCollection(metaRoot); + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(offlineConfigProviders !== undefined ? { providers: offlineConfigProviders } : {}), }); } catch (err) { @@ -889,6 +1071,19 @@ export async function runOfflineGenerate( return 2; } + const offlineDialect = config.dialect; + const offlineViews = buildProjectionViews(metadata, { dialect: offlineDialect, columnNamingStrategy: offlineStrategy }); + const scopeRc = refuseScopeMismatch( + collection, + () => buildExpectedSchemaWithProvenance(metadata, { + dialect: offlineDialect, + columnNamingStrategy: offlineStrategy, + views: offlineViews, + }).provenance, + fmt, + ); + if (scopeRc !== undefined) return scopeRc; + const outDir = resolvePath(metaRoot, config.outDir); const path = snapshotPath(outDir, config.dialect); let snapshot; @@ -916,7 +1111,7 @@ export async function runOfflineGenerate( const collectedAmbiguous: AmbiguousChange[] = []; const onAmbiguousResolution = mapOnAmbiguous(config.onAmbiguous); - const offlineViews = buildProjectionViews(metadata, { dialect: config.dialect, columnNamingStrategy: offlineStrategy }); + const offlineScope = collection.inMigrateScope; let plan; try { @@ -926,6 +1121,8 @@ export async function runOfflineGenerate( snapshot, columnNamingStrategy: offlineStrategy, views: offlineViews, + // Per-command scope — narrows BOTH sides of the offline diff (see planOffline). + ...(offlineScope !== undefined ? { inScope: offlineScope } : {}), allow: tokensToAllowOptions(config.allow), onAmbiguous: async (a) => { collectedAmbiguous.push(a); @@ -946,7 +1143,11 @@ export async function runOfflineGenerate( throw err; } - const { diff: diffResult, nextSnapshot } = plan; + // `nextSnapshot` is what gets COMMITTED (it retains this run's out-of-scope + // entries); `expected` is the governed side the emitter renders against. Equal + // for an unscoped run. + const { diff: diffResult, nextSnapshot, expected: governedExpected } = plan; + logOutOfScope(plan.outOfScope, fmt); if (diffResult.blocked.length > 0) { log.error(`migrate: ${diffResult.blocked.length} destructive change(s) blocked; re-run with --allow `); @@ -963,7 +1164,7 @@ export async function runOfflineGenerate( const emitResult = emit(diffResult.changes, { dialect: config.dialect, - expectedSchema: nextSnapshot, + expectedSchema: governedExpected, actualSchema: snapshot, ...(snapshot.meta ? { actualMeta: snapshot.meta } : {}), }); @@ -1052,6 +1253,7 @@ async function runRollback( async function runD1Migrate( config: ResolvedMigrateConfig, metaRoot: string, + cwd: string, runner: WranglerRunner, fmt: OutputFormat = "text", ): Promise { @@ -1079,6 +1281,12 @@ async function runD1Migrate( binding = { binding: config.d1.binding!, database_name: "", database_id: "", migrations_dir: undefined }; } + // The binding — and with it wrangler.toml's `migrations_dir` — is only now + // known, so this is the earliest point d1 can honestly answer "where will + // this run write?" (the caller skipped its own generic check for exactly + // this reason; see the guard around that call). + warnIfLedgerRelocated(cwd, resolveD1OutDir(config, metaRoot, binding.migrations_dir)); + // 2. Build a D1Runner closure over the wrangler runner. const d1Runner: D1Runner = async (sql) => { const args = buildWranglerExecuteArgs({ @@ -1101,18 +1309,25 @@ async function runD1Migrate( d1ConfigProviders = undefined; } + // Discovery and load are separate failure modes (the `meta gen` pattern); + // `resolveCollection`'s own ERR_COLLECTION_NOT_FOUND replaces the hand-rolled + // ENOENT sniff, with the same exit 2. + let collection; + try { + collection = await resolveCollection(metaRoot); + } catch (err) { + log.error((err as Error).message); + return 2; + } + let metadata; try { - metadata = await loadMemory(metaRoot, { + metadata = await loadMemory(collection.configDir, { + files: collection.files, ...(d1ConfigProviders !== undefined ? { providers: d1ConfigProviders } : {}), }); } catch (err) { - const msg = (err as Error).message; - if (msg.includes("ENOENT") || msg.includes("no such") || msg.includes("cannot read")) { - log.error(`no metaobjects/ found in ${metaRoot}; run 'meta init' to scaffold`); - } else { - log.error(`migrate: failed to load metadata: ${msg}`); - } + log.error(`migrate: failed to load metadata: ${(err as Error).message}`); return 2; } @@ -1125,7 +1340,13 @@ async function runD1Migrate( // metaobjects.config.ts absent or invalid — use default snake_case } const expectedViews = buildProjectionViews(metadata, { dialect: "d1", columnNamingStrategy }); - const expected = buildExpectedSchema(metadata, { dialect: "d1", columnNamingStrategy, views: expectedViews }); + const built = buildExpectedSchemaWithProvenance(metadata, { dialect: "d1", columnNamingStrategy, views: expectedViews }); + const scopeRc = refuseScopeMismatch(collection, () => built.provenance, fmt); + if (scopeRc !== undefined) return scopeRc; + // Per-command scope — both-sided, exactly as on the Kysely path above. + const scoped = scopeExpectedSchema(built, collection.inMigrateScope); + const expected = scoped.snapshot; + logOutOfScope(scoped.outOfScope, fmt); let actual; try { actual = await introspectD1({ @@ -1145,7 +1366,8 @@ async function runD1Migrate( let diffResult; try { diffResult = await diff({ - expected, + // The three scoped-diff obligations, exactly as on the online path above. + ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, // D1 is SQLite at the SQL level — the dialect activates the sqlite diff // semantics (structural FK matching: SQLite stores no FK names; CHECK @@ -1157,8 +1379,6 @@ async function runD1Migrate( // has no expressible migration; refuse loudly instead of emitting SQL that drops // the constraint and breaks referencing FKs at apply (same failure as the online path). refusePrimaryKeyChange: true, - // #208 §7 — declared-@unmanaged objects are external (see the online path above). - unmanagedNames: collectUnmanagedNames(metadata), onAmbiguous: async (a) => { collectedAmbiguous.push(a); return onAmbiguousResolution; @@ -1217,14 +1437,9 @@ async function runD1Migrate( const combinedUp = emitResult.up; const combinedDown = emitResult.down; - // Migration dir resolution: --out-dir > wrangler.toml's migrations_dir > "migrations". - // The default outDir (./.metaobjects/migrations) is the Kysely-path default; for D1 - // we fall back to wrangler conventions when the caller hasn't overridden it. - const isDefaultOutDir = config.outDir === MIGRATE_DEFAULT_OUT_DIR; - const migrationsDir = resolvePath( - metaRoot, - isDefaultOutDir ? (binding.migrations_dir ?? "migrations") : config.outDir, - ); + // Migration dir resolution — same convention `warnIfLedgerRelocated` was + // just given above, so the two cannot drift apart. + const migrationsDir = resolveD1OutDir(config, metaRoot, binding.migrations_dir); if (config.dryRun) { log.info(`-- UP --\n${combinedUp}\n\n-- DOWN --\n${combinedDown}`); diff --git a/server/typescript/packages/cli/src/commands/prompt-snapshot.ts b/server/typescript/packages/cli/src/commands/prompt-snapshot.ts index d5978f69f..be80b7b22 100644 --- a/server/typescript/packages/cli/src/commands/prompt-snapshot.ts +++ b/server/typescript/packages/cli/src/commands/prompt-snapshot.ts @@ -15,7 +15,7 @@ import { log } from "../lib/log.js"; import { FileProvider } from "../lib/file-provider.js"; import { snapshotPaths, unifiedDiff } from "../lib/snapshot.js"; import { loadMetaobjectsConfig } from "../lib/load-metaobjects-config.js"; -import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; import { TYPE_TEMPLATE, TEMPLATE_ATTR_TEXT_REF, TEMPLATE_ATTR_FORMAT } from "@metaobjectsdev/metadata"; import { render, ESCAPERS, type RenderFormat } from "@metaobjectsdev/render"; @@ -30,13 +30,36 @@ export async function promptSnapshotCommand(args: string[], cwd: string): Promis return 2; } + // Where the metadata lives is `resolveCollection`'s decision, not this + // command's — `--check` is a drift GATE, so a project declaring `sources` + // elsewhere would otherwise gate against a stale `metaobjects/` (or report + // "no metaobjects/ found" for metadata it can see perfectly well). Discovery + // and load stay separate failure modes, the `meta gen` pattern: a broad catch + // around both reports a genuine ParseError as "no metadata found". + // `resolveCollection` raises ERR_COLLECTION_NOT_FOUND with its own message, + // replacing the hand-rolled ENOENT sniff that used to live here. + let collection; + try { + collection = await resolveCollection(cwd); + } catch (err) { + log.error((err as Error).message); + return 2; + } + + // Everything project-relative below hangs off the DECLARING directory, not + // ambient cwd: `.metaobjects/snapshots/` is that config's own state, and the + // prompt text belongs to the same project as the metadata that references it. + // Identical to cwd for a run from the project root, which is the only + // invocation that worked before metadata sources were resolvable at all. + const projectRoot = collection.configDir; + // Best-effort load of metaobjects.config.ts to pick up consumer-supplied // providers. prompt-snapshot doesn't require codegen config; if it's absent // or invalid, fall back to defaults — the loader still works for any // metadata that only uses core+forge subtypes. let configProviders: NonNullable>["providers"]> | undefined; try { - const forgeConfig = await loadMetaobjectsConfig(cwd); + const forgeConfig = await loadMetaobjectsConfig(projectRoot); configProviders = forgeConfig.providers; } catch { configProviders = undefined; @@ -44,20 +67,16 @@ export async function promptSnapshotCommand(args: string[], cwd: string): Promis let root; try { - root = await loadMemory(cwd, { + root = await loadMemory(collection.configDir, { + files: collection.files, ...(configProviders !== undefined ? { providers: configProviders } : {}), }); } catch (err) { - const msg = (err as Error).message; - if (msg.includes("ENOENT") || msg.includes("no such") || msg.includes("cannot read")) { - log.error(`no metaobjects/ found in ${cwd}; run 'meta init' to scaffold`); - return 2; - } - log.error(`failed to load metadata: ${msg}`); + log.error(`failed to load metadata: ${(err as Error).message}`); return 1; } - const promptsDir = join(cwd, flags.prompts ?? DEFAULT_PROMPTS_DIR); + const promptsDir = join(projectRoot, flags.prompts ?? DEFAULT_PROMPTS_DIR); const provider = new FileProvider(promptsDir); // ADR-0039: effective children — resolve rather than rely on root being unextended. @@ -79,7 +98,7 @@ export async function promptSnapshotCommand(args: string[], cwd: string): Promis // Absent/typeless required attrs are a loader-schema concern, not ours. if (typeof textRef !== "string") continue; - const { dir, payloadPath, snapPath } = snapshotPaths(cwd, tmpl.name); + const { dir, payloadPath, snapPath } = snapshotPaths(projectRoot, tmpl.name); if (!existsSync(payloadPath)) { log.info(`[${tmpl.name}] skipped — no payload at ${payloadPath}`); skipped++; diff --git a/server/typescript/packages/cli/src/commands/verify.ts b/server/typescript/packages/cli/src/commands/verify.ts index bf55fb5d8..02d6cfe5b 100644 --- a/server/typescript/packages/cli/src/commands/verify.ts +++ b/server/typescript/packages/cli/src/commands/verify.ts @@ -27,12 +27,16 @@ import { } from "../lib/wrangler.js"; import type { MetaobjectsGenConfig } from "@metaobjectsdev/codegen-ts"; import { buildProjectionViews } from "@metaobjectsdev/codegen-ts"; -import { buildKyselyFromUrl, type Dialect } from "../lib/kysely.js"; +import { buildKyselyFromUrl, inferDialect, type Dialect } from "../lib/kysely.js"; import { tokensToAllowOptions, describeChange } from "../lib/allow.js"; import { computeDrift, computeDriftFromActual, collectUnmanagedNames, + excludeFromSnapshot, + scopedDiffInputs, + buildExpectedSchemaWithProvenance, + type GovernedScope, introspect, diff, readSnapshot, @@ -45,9 +49,10 @@ import { type Change, type D1Binding, type D1Runner, - type DiffResult, + type DriftResult, } from "@metaobjectsdev/migrate-ts"; -import { loadMemory } from "@metaobjectsdev/sdk"; +import { loadMemory, resolveCollection } from "@metaobjectsdev/sdk"; +import { migrateScopeMismatch, outOfScopeNote } from "../lib/migrate-scope.js"; import { TYPE_TEMPLATE, TEMPLATE_SUBTYPE_PROMPT, @@ -107,9 +112,6 @@ export async function verifyCommand( return 2; } - // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. - warnIfAgentContextStale(cwd); - // ADR-0021 D2 — explicit verify subverbs. Each flag selects one drift mode; // any combination runs each and the overall exit code is the MAX (non-zero on // any drift). A bare `verify` (no explicit subverb) keeps its documented @@ -126,6 +128,42 @@ export async function verifyCommand( ); } + // Where the metadata lives is `resolveCollection`'s decision, not a hardcoded + // directory. It also carries the per-command `migrate.scope` the schema gate below + // honours — `verify --db` and `migrate` govern the identical object set — and the + // top-level `scope` `runCodegenVerify` (a nested function below) threads into + // `computeCodegenDrift`. Explicitly typed (unlike the `let collection;` pattern + // elsewhere in this codebase): a nested function body is OUTSIDE the control-flow + // narrowing TS performs on a same-scope `let x;` reassignment, so a bare + // `let collection;` type-checked clean until this task added exactly that nested + // reference — the reader who removes the annotation next reintroduces TS7034. + let collection: Awaited>; + try { + collection = await resolveCollection(cwd); + } catch (err) { + log.error((err as Error).message); + return 2; + } + + // The project root is whichever directory `resolveCollection` decided the + // metadata belongs to (design §4.6.1: "Per-port generator config is then read + // from that same directory"). The line this draws, applied throughout this + // command: anything named BY the metadata or its config resolves against + // `projectRoot` — `metaobjects.config.ts`, `.metaobjects/config.json`, the + // `outDir` and `wranglerConfigPath` they carry, the `prompts/` a `@textRef` + // resolves in, the test files a `@verifiedBy` names. Identical paths for a run + // from the project root. + // + // The two advisory passes — the agent-context staleness nudge and the + // anti-pattern scan — are rooted here too, matching `meta gen`. Both commands + // describe them as the same pass, and scanning two different trees for it made + // that false: a `verify` run from a subdirectory scanned only that subtree and + // found no agent-context manifest at all, so the nudge silently never fired. + const projectRoot = collection.configDir; + + // Advisory: nudge to refresh the .claude/skills docs if they predate this CLI. + warnIfAgentContextStale(projectRoot); + // Best-effort load of metaobjects.config.ts. Two consumers: // 1) consumer-supplied providers (e.g. a `template.toolcall` subtype) threaded // into loadMemory — verify doesn't REQUIRE codegen config for templates/db; @@ -134,7 +172,7 @@ export async function verifyCommand( // error (it can't diff without knowing where the committed output lives). let forgeConfig: MetaobjectsGenConfig | undefined; try { - forgeConfig = await loadMetaobjectsConfig(cwd); + forgeConfig = await loadMetaobjectsConfig(projectRoot); } catch { forgeConfig = undefined; } @@ -144,16 +182,13 @@ export async function verifyCommand( // so an undeclared/typo'd own @attr fails verify (matching Java's Maven goal). let root: Awaited>; try { - root = await loadMemory(cwd, { + root = await loadMemory(collection.configDir, { + files: collection.files, ...(configProviders !== undefined ? { providers: configProviders } : {}), strict: !flags.lax, }); } catch (err) { const msg = (err as Error).message; - if (msg.includes("ENOENT") || msg.includes("no such") || msg.includes("cannot read")) { - log.error(`no metaobjects/ found in ${cwd}; run 'meta init' to scaffold`); - return 2; - } log.error(`failed to load metadata: ${msg}`); // Strict-load rejection (ADR-0023): give the author the three exits — register // the attr on a provider, stash it in the `attr.properties` bag, or pass --lax. @@ -169,7 +204,13 @@ export async function verifyCommand( return 1; } - const promptsDir = join(cwd, flags.prompts ?? DEFAULT_PROMPTS_DIR); + // The schema gate governs exactly the objects `meta migrate` governs — ONE + // declaration (`migrate.scope`), not a second key: a drift gate that fails on + // tables migrate deliberately does not own is incoherent. Undefined ⇒ everything + // loaded, which is every project that declares no scope. + const schemaScope = collection.inMigrateScope; + + const promptsDir = join(projectRoot, flags.prompts ?? DEFAULT_PROMPTS_DIR); const provider = new FileProvider(promptsDir); // Exit-code composition: the overall result is the MAX across every selected @@ -200,7 +241,7 @@ export async function verifyCommand( // authority — see the verified-by-scan header. const diags = [ ...checkRequirements(root), - ...checkVerifiedBy(root, cwd, forgeConfig?.verify?.testFiles), + ...checkVerifiedBy(root, projectRoot, forgeConfig?.verify?.testFiles), ]; // Printed on EVERY run, clean or not — a gate that says nothing when it @@ -245,7 +286,7 @@ export async function verifyCommand( function runAntiPatternAdvisory(): void { let findings; try { - findings = scanSourceForAntiPatterns(cwd); + findings = scanSourceForAntiPatterns(projectRoot); } catch { return; // never let an advisory scan break verify } @@ -378,6 +419,26 @@ export async function verifyCommand( const usingD1 = flags.dialect === "d1"; if ((flags.db === undefined && !usingD1) || flags.skipSchema) return 0; + // A `migrate.scope` matching nothing it could govern is refused, not tolerated — + // it would make this gate compare zero objects and report "in sync" (see + // `migrateScopeMismatch`). Checked HERE rather than beside the other collection + // work at the top of `verifyCommand`, because `migrate.scope` governs only the + // schema gate: a stale pattern must not fail a `--templates` run that never + // consults it. + const scopeMismatch = migrateScopeMismatch(collection, () => { + const dialect: Dialect = usingD1 ? "d1" : (flags.dialect ?? inferDialect(flags.db as string)); + const viewStrategy = forgeConfig?.columnNamingStrategy ?? "snake_case"; + return buildExpectedSchemaWithProvenance(root, { + dialect, + columnNamingStrategy: viewStrategy, + views: buildProjectionViews(root, { dialect, columnNamingStrategy: viewStrategy }), + }).provenance; + }); + if (scopeMismatch !== undefined) { + log.error(`verify: ${scopeMismatch}`); + return 2; + } + if (usingD1 && flags.db !== undefined) { log.error(`verify: --db is not used for dialect 'd1' — wrangler.toml owns the connection; pass --d1 instead`); return 2; @@ -428,7 +489,11 @@ export async function verifyCommand( // `actual` this drift comparison uses, and re-introspecting for it would both // cost a second round trip and open a window where the two could disagree. actual = await introspect(kysely.db, kysely.dialect); - driftResult = await computeDriftFromActual(actual, kysely.dialect, root, { allow, views: expectedViews }); + driftResult = await computeDriftFromActual(actual, kysely.dialect, root, { + allow, + views: expectedViews, + ...(schemaScope !== undefined ? { inScope: schemaScope } : {}), + }); } catch (err) { log.error(`verify: failed to introspect ${kysely.displayUrl}: ${(err as Error).message}`); return 1; @@ -436,7 +501,7 @@ export async function verifyCommand( const snapshotDrift = driftResult.changes.length === 0 - ? await checkCommittedSnapshot(actual, kysely.dialect, kysely.displayUrl) + ? await checkCommittedSnapshot(actual, kysely.dialect, kysely.displayUrl, driftResult) : []; return reportSchemaDrift(driftResult, [...ledgerDrift, ...snapshotDrift], kysely.displayUrl); @@ -460,14 +525,14 @@ export async function verifyCommand( // computeDriftFromActual and the SAME reportSchemaDrift the sqlite/postgres // path uses — no forked reporting/exit-code logic. async function runD1SchemaVerify(ledgerDrift: string[]): Promise { - const d1Config = await resolveD1Config({ d1Binding: flags.d1, remote: flags.remote }, cwd); + const d1Config = await resolveD1Config({ d1Binding: flags.d1, remote: flags.remote }, projectRoot); const wranglerConfigPath = d1Config.wranglerConfigPath - ? resolvePath(cwd, d1Config.wranglerConfigPath) - : findWranglerConfig(cwd); + ? resolvePath(projectRoot, d1Config.wranglerConfigPath) + : findWranglerConfig(projectRoot); if (wranglerConfigPath === undefined && d1Config.binding === undefined) { - log.error(`verify: no wrangler.toml found in ${cwd} or parents; pass --d1 to bypass`); + log.error(`verify: no wrangler.toml found in ${projectRoot} or parents; pass --d1 to bypass`); return 2; } @@ -493,7 +558,7 @@ export async function verifyCommand( command: sql, configPath: wranglerConfigPath, }); - const { stdout } = await activeWranglerRunner(wranglerArgs, cwd); + const { stdout } = await activeWranglerRunner(wranglerArgs, projectRoot); return stdout; }; @@ -510,7 +575,11 @@ export async function verifyCommand( const expectedViews = buildProjectionViews(root, { dialect: "d1", columnNamingStrategy: viewStrategy }); let driftResult; try { - driftResult = await computeDriftFromActual(actual, "d1", root, { allow, views: expectedViews }); + driftResult = await computeDriftFromActual(actual, "d1", root, { + allow, + views: expectedViews, + ...(schemaScope !== undefined ? { inScope: schemaScope } : {}), + }); } catch (err) { log.error(`verify: ${(err as Error).message}`); return 1; @@ -554,14 +623,15 @@ export async function verifyCommand( actual: SchemaSnapshot, dialect: Dialect, displayUrl: string, + governed: GovernedScope, ): Promise { if (dialect === "d1") return []; // d1 keeps migrations Wrangler-native; no offline snapshot // Resolve the migrations dir through migrate's OWN precedence (flag > config > // default) rather than re-deriving it, so verify can never look somewhere migrate // does not write. Only `outDir` is consumed; the rest of the resolved config is // migrate's business. - const migrateConfig = await resolveMigrateConfig(EMPTY_MIGRATE_FLAGS, cwd); - const dir = resolvePath(cwd, migrateConfig.outDir); + const migrateConfig = await resolveMigrateConfig(EMPTY_MIGRATE_FLAGS, projectRoot); + const dir = resolvePath(projectRoot, migrateConfig.outDir); let snapshot: SchemaSnapshot | null; try { snapshot = await readSnapshot(snapshotPath(dir, dialect)); @@ -570,11 +640,19 @@ export async function verifyCommand( } if (snapshot === null) return []; + // Out-of-scope objects leave BOTH sides of this comparison, and the schema pin + // comes from the scope decision the DRIFT comparison already made — one door + // (migrate-ts's `excludeFromSnapshot` + `scopedDiffInputs`), not a fifth + // hand-rolled copy of the three-part contract. `unmanagedNames` suppresses the + // actual side only, which is right for the metadata↔DB diff (its expected side + // is already scoped) but not here: the committed snapshot IS the expected side, + // and a snapshot written before the scope was declared still carries the other + // owner's tables. Re-deriving the pin from the snapshot is what left an empty + // (never-migrated) snapshot reaching `diff`'s whole-database fallback. const result = await diff({ - expected: snapshot, + ...scopedDiffInputs(excludeFromSnapshot(snapshot, governed), collectUnmanagedNames(root)), actual, allow: {}, - unmanagedNames: collectUnmanagedNames(root), }); if (result.changes.length === 0) return []; @@ -586,7 +664,7 @@ export async function verifyCommand( ]; } - function reportSchemaDrift(driftResult: DiffResult, ledgerDrift: string[], displayUrl: string): number { + function reportSchemaDrift(driftResult: DriftResult, ledgerDrift: string[], displayUrl: string): number { // #208 §8 — make declared-external objects visible: they are excluded from the // drift comparison (computeDrift/computeDriftFromActual thread them out), so // annotate them as external (declared) rather than let them vanish silently. @@ -597,6 +675,13 @@ export async function verifyCommand( ); } + // Same reasoning for the per-command scope: an object `migrate.scope` excluded + // was NOT checked, and silence would misreport it as checked-and-clean. Shared + // wording with `meta migrate` — one declaration, one sentence about it. + if (driftResult.outOfScope.length > 0) { + log.info(outOfScopeNote("verify", driftResult.outOfScope)); + } + const changes = driftResult.changes; if (changes.length === 0 && ledgerDrift.length === 0) { log.info(`meta verify — schema in sync with ${displayUrl}.`); @@ -629,9 +714,13 @@ export async function verifyCommand( return 2; } + // The identical predicate `meta gen` applies (Task 12b / design §7 open + // question 3) — a `gen` that committed under a narrowed scope and a + // `verify --codegen` that regenerates unscoped would disagree about which + // files should exist, reporting every out-of-scope entity as drift. let result; try { - result = await computeCodegenDrift(forgeConfig, root, cwd); + result = await computeCodegenDrift(forgeConfig, root, projectRoot, collection.inScope); } catch (err) { log.error(`verify --codegen: regeneration failed: ${(err as Error).message}`); return 1; diff --git a/server/typescript/packages/cli/src/index.ts b/server/typescript/packages/cli/src/index.ts index dac469d0d..24cda0466 100644 --- a/server/typescript/packages/cli/src/index.ts +++ b/server/typescript/packages/cli/src/index.ts @@ -2,6 +2,7 @@ import { resolve } from "node:path"; import { log } from "./lib/log.js"; import { cliVersion } from "./lib/version.js"; import { resolveFormat, isValidFormat, VALID_FORMATS } from "./lib/format.js"; +import { resolveCollection } from "@metaobjectsdev/sdk"; export { defineConfig } from "@metaobjectsdev/codegen-ts"; export type { MetaobjectsGenConfig } from "@metaobjectsdev/codegen-ts"; @@ -13,10 +14,10 @@ USAGE: meta [flags] COMMANDS: - init Scaffold metaobjects/ + .metaobjects/ in the current repo + init Scaffold a MetaObjects project in the current repo init --refresh-docs Refresh .metaobjects/AGENTS.md + CLAUDE.md after CLI upgrades agent-docs Scaffold only the agent-context (.metaobjects/ + .claude/skills/) — canonical redirect target for all language ports - gen [...] Codegen TS targets from metaobjects/ entities + gen [...] Codegen TS targets from your declared metadata types [query] Search the metadata vocabulary (types, subtypes, @attrs) by name or description export Flatten loaded metadata to one canonical JSON artifact docs --out Generate neutral metadata documentation (entity + template pages; --site for HTML site) @@ -39,7 +40,7 @@ EXPORT FLAGS: --out Write output to a file (default: stdout) DOCS FLAGS: - Project root holding metaobjects/ (default: current directory) + Project root to resolve metadata from (default: current directory) --out , -o Output directory for the pages (default: ./docs) --templates Project root to resolve adopter templates/ overrides (default: ) --prompts Extra dir holding prompt .mustache sources for --site (e.g. data/templates/) @@ -90,7 +91,7 @@ ship in later sub-projects. See https://metaobjects.com for docs. /** Focused per-subcommand usage slices shown by ` --help`. */ const COMMAND_HELP: Record = { - gen: `meta gen — codegen TS targets from metaobjects/ entities + gen: `meta gen — codegen TS targets from your declared metadata USAGE: meta gen [...] [flags] @@ -152,7 +153,7 @@ USAGE: meta docs [] [flags] FLAGS: - Project root holding metaobjects/ (default: current directory) + Project root to resolve metadata from (default: current directory) --out , -o Output directory for the pages (default: ./docs) --model Emit the markdown model surface (entity + template pages) --api Emit the markdown api surface (generated SDK reference) @@ -161,10 +162,10 @@ FLAGS: --scaffold-site Copy the site's templates + assets into codegen/docs-site/ to own (theme) them --templates Project root to resolve adopter templates/ overrides (default: ) --prompts Extra dir holding prompt .mustache sources (for --site) when they - live outside metaobjects/ or templates/ (e.g. data/templates/) + live outside the metadata sources or templates/ (e.g. data/templates/) --help, -h Print this help `, - init: `meta init — scaffold metaobjects/ + .metaobjects/ in the current repo + init: `meta init — scaffold a MetaObjects project in the current repo USAGE: meta init [flags] @@ -271,12 +272,19 @@ export async function run(argv: string[]): Promise { case undefined: { // Content-first no-args view: concise status + next-step help[] rather than // dumping the full manual (full manual is still available via `meta --help`). - const metaobjectsExists = await import("node:fs/promises") - .then(({ stat }) => stat(resolve(cwd, "metaobjects")).then(() => true).catch(() => false)); - const statusLine = metaobjectsExists - ? `meta — MetaObjects CLI (v${VERSION}) · metaobjects/ found` - : `meta — MetaObjects CLI (v${VERSION}) · no metaobjects/ here`; - const nextSteps = metaobjectsExists + // "Is this a MetaObjects project?" routes through resolveCollection — the + // single authority on where metadata lives — rather than assuming the + // default `metaobjects/` directory name. The status line must not assert + // it either: a project whose config points `sources` at + // `../shared-model/metadata` would be told "metaobjects/ found", which is + // false, and one that resolves nothing would be told there is no + // `metaobjects/` here when the real problem may be a declared source that + // failed to resolve. + const metadataResolves = await resolveCollection(cwd).then(() => true).catch(() => false); + const statusLine = metadataResolves + ? `meta — MetaObjects CLI (v${VERSION}) · metadata found` + : `meta — MetaObjects CLI (v${VERSION}) · no MetaObjects project here`; + const nextSteps = metadataResolves ? [ " meta gen Run codegen", " meta verify Check for drift", @@ -284,7 +292,7 @@ export async function run(argv: string[]): Promise { " meta --help Full command reference", ] : [ - " meta init Scaffold metaobjects/ in this directory", + " meta init Scaffold a MetaObjects project in this directory", " meta --help Full command reference", ]; log.info(`${statusLine}\n\n${nextSteps.join("\n")}\n`); diff --git a/server/typescript/packages/cli/src/lib/codegen-drift.ts b/server/typescript/packages/cli/src/lib/codegen-drift.ts index 7ce47bb58..70d921fa6 100644 --- a/server/typescript/packages/cli/src/lib/codegen-drift.ts +++ b/server/typescript/packages/cli/src/lib/codegen-drift.ts @@ -81,11 +81,19 @@ function listFiles(dir: string): string[] { * @param config the loaded metaobjects config (provides outDir/targets). * @param metadata the loaded MetaRoot (same object `meta gen` would use). * @param projectRoot absolute project root (committed outDirs are keyed off it). + * @param scope the SAME output-scope predicate `meta gen` used to produce the + * committed output (Task 12b / design §7 open question 3). A `verify --codegen` + * that regenerates unscoped while the committed output was scoped would read + * every out-of-scope entity as drift — regen would try to emit it, but it was + * never committed because the `meta gen` that produced the committed tree + * never emitted it either. Undefined ⇒ everything is in scope (byte-identical + * to a project with no `scope` declared). */ export async function computeCodegenDrift( config: MetaobjectsGenConfig, metadata: MetaData, projectRoot: string, + scope?: (fqn: string) => boolean, ): Promise { const root = isAbsolute(projectRoot) ? projectRoot : resolve(projectRoot); @@ -157,6 +165,7 @@ export async function computeCodegenDrift( genStateDir: join(tempRoot, ".gen-state"), mergeStrategy: "overwrite", baseline: "fresh", + ...(scope !== undefined ? { scope } : {}), }); // Diff each committed outDir against its temp mirror. diff --git a/server/typescript/packages/cli/src/lib/detect-stack.ts b/server/typescript/packages/cli/src/lib/detect-stack.ts index 074a44853..2ff6c5eaa 100644 --- a/server/typescript/packages/cli/src/lib/detect-stack.ts +++ b/server/typescript/packages/cli/src/lib/detect-stack.ts @@ -1,6 +1,6 @@ import { existsSync, readFileSync, readdirSync } from "node:fs"; -import type { Dirent } from "node:fs"; import { join } from "node:path"; +import { resolveCollection } from "@metaobjectsdev/sdk"; import { detectStack, detectConcerns, makeStack, type ServerLang, type ClientFramework, type Stack, type ProjectProbe, @@ -21,62 +21,49 @@ function depNames(cwd: string): Set { return out; } -const METADATA_DIR = "metaobjects"; -const METADATA_FILE_PATTERN = /\.(json|ya?ml)$/i; // Cheap substring probe, not a metamodel load: matches both canonical JSON's // quoted `"requirement.functional"` key and sigil-free YAML's bare // `requirement.functional:` authoring form. const REQUIREMENT_NODE_MARKER = "requirement."; -/** Recursively scans `metaobjects/` for any `.json`/`.yaml`/`.yml` file containing a - * `requirement.*` node marker. Defensive throughout: a missing/unreadable directory - * or file is treated as "not found", never thrown — this is a cheap heuristic, not - * a metamodel load. */ -function hasRequirementNodes(cwd: string): boolean { - const root = join(cwd, METADATA_DIR); - if (!existsSync(root)) return false; - const pending: string[] = [root]; - while (pending.length > 0) { - const dir = pending.pop()!; - let entries: Dirent[]; - try { - entries = readdirSync(dir, { withFileTypes: true }); - } catch { - continue; // unreadable directory — skip it, keep scanning siblings - } - for (const entry of entries) { - const full = join(dir, entry.name); - if (entry.isDirectory()) { - pending.push(full); - } else if (METADATA_FILE_PATTERN.test(entry.name)) { - try { - if (readFileSync(full, "utf8").includes(REQUIREMENT_NODE_MARKER)) return true; - } catch { /* unreadable file — treat as no match */ } - } +/** Scans the project's resolved metadata collection (`resolveCollection` — the + * single authority on where metadata lives, honouring declared `sources` rather + * than assuming `metaobjects/`) for any file containing a `requirement.*` node + * marker. Defensive throughout: no declared sources and no default directory, an + * unresolvable source, or an unreadable file are all treated as "not found", + * never thrown — this is a cheap heuristic, not a metamodel load. */ +async function hasRequirementNodes(cwd: string): Promise { + try { + const { files } = await resolveCollection(cwd); + for (const file of files) { + if (readFileSync(file, "utf8").includes(REQUIREMENT_NODE_MARKER)) return true; } + return false; + } catch { + return false; } - return false; } -function probe(cwd: string): ProjectProbe { +async function probe(cwd: string): Promise { const deps = depNames(cwd); const names = existsSync(cwd) ? readdirSync(cwd) : []; + const requirementNodes = await hasRequirementNodes(cwd); return { hasDep: (name) => deps.has(name), hasFileMatching: (re) => names.some((n) => re.test(n)), - hasRequirementNodes: () => hasRequirementNodes(cwd), + hasRequirementNodes: () => requirementNodes, }; } /** Resolve the stack: explicit --server/--client overrides take precedence; otherwise detect. * Concern tokens (e.g. requirements) are always OBSERVED from project state, independent of * any --server/--client override — a concern is not a stack axis. */ -export function resolveStack(cwd: string, overrides: { servers: string[]; clients: string[] }): Stack { +export async function resolveStack(cwd: string, overrides: { servers: string[]; clients: string[] }): Promise { const validServers = SERVER_LANGS as readonly string[]; const validClients = CLIENT_FRAMEWORKS as readonly string[]; const oServers = overrides.servers.filter((s): s is ServerLang => validServers.includes(s)); const oClients = overrides.clients.filter((c): c is ClientFramework => validClients.includes(c)); - const p = probe(cwd); + const p = await probe(cwd); const concerns = detectConcerns(p); if (oServers.length > 0 || oClients.length > 0) return makeStack(oServers, oClients, concerns); const detected = detectStack(p); diff --git a/server/typescript/packages/cli/src/lib/migrate-scope.ts b/server/typescript/packages/cli/src/lib/migrate-scope.ts new file mode 100644 index 000000000..8d2d814b2 --- /dev/null +++ b/server/typescript/packages/cli/src/lib/migrate-scope.ts @@ -0,0 +1,102 @@ +// What `meta migrate` and `meta verify --db` SAY about a declared `migrate.scope`. +// +// The scope itself needs no adapter: `resolveCollection` hands back +// `inMigrateScope` already in migrate-ts's predicate shape, so the pattern grammar +// lives in exactly one place (`matchesScope`, @metaobjectsdev/sdk) and `migrate` and +// `gen` cannot come to disagree about what `acme::platform::**` means. +// +// What DOES need one home is the user-facing language, and both commands import it +// from here: they govern the identical object set from one declaration, so a note +// or a refusal that drifted between them would be drift the user reads. + +import type { Collection } from "@metaobjectsdev/sdk"; +import type { SchemaProvenance } from "@metaobjectsdev/migrate-ts"; + +/** + * Say what a declared `migrate.scope` left out, for `migrate` and `verify --db` + * alike. + * + * An excluded object produces neither a create nor a drop and is neither checked + * nor reported as drift, so without this line "no changes" and "no changes to the + * half of the model this run governs" read identically — and an unchecked table is + * indistinguishable from a checked-and-clean one. + * + * One sentence, one definition: the two commands say the same thing about the same + * declaration, and this is a string a user reads, so drift between two copies of it + * is drift the user sees. + */ +export function outOfScopeNote(command: string, names: readonly string[]): string { + return ( + `meta ${command} — ${names.length} object(s) out-of-scope ` + + `(outside migrate.scope, governed elsewhere): ${names.join(", ")}` + ); +} + +/** How many loaded FQNs to name in the refusal below — enough to show the shape + * an author's patterns have to match, short enough to stay readable. */ +const EXAMPLE_FQN_CAP = 3; + +/** + * The refusal for a `migrate.scope` that matches NOTHING it could govern. + * + * A scope matching zero of the objects that declare a table or view can never be + * what someone meant — it says "every table in this model belongs to somebody + * else", which is a project with no schema to migrate at all, expressed the hard + * way. In practice it is a typo'd or stale package pattern, or a scope over a + * package that holds only value objects and abstracts (shapes that can never + * contribute a table or view), and it is silent: migrate reports "no changes" + * while having compared nothing. + * + * It is also actively dangerous, which is why this is a refusal and not a warning. + * An empty expected side is what `diff` reads as "no model, govern the whole + * database" — the inversion `scopeExpectedSchema`'s `declaredSchemas` closes + * structurally (migrate-ts `scope.ts`). This is the second lock on the same door: + * the structural fix stops a wrong scope proposing a destructive change, and this + * stops the wrong scope going unnoticed in the first place. + * + * Returns the message to report, or `undefined` when there is nothing to refuse — + * no scope declared, or at least one table- or view-declaring object inside it. + * Callers report it and exit 2 (a configuration error), rather than this throwing, + * so it reads like every other config failure in these commands. + */ +export function migrateScopeMismatch( + collection: Collection, + /** + * The UNSCOPED expected schema's provenance (migrate-ts + * `buildExpectedSchemaWithProvenance`) — qualified table/view name → declaring + * FQN. Supplied lazily because it is consulted only under a declared scope, so + * a project with no `migrate.scope` pays nothing for this check and its runs + * are byte-for-byte what they always were. + */ + provenance: () => SchemaProvenance, +): string | undefined { + const { inMigrateScope, migrateScopePatterns } = collection; + if (inMigrateScope === undefined) return undefined; + + // The declaring FQNs of every table and view the UNSCOPED model contributes — + // the same provenance `scopeExpectedSchema` decides scope on, so the refusal + // asks exactly the question the run answers. NOT the loaded object set: that + // counts value objects and abstracts, which can never declare a table or view + // (persistability derives from a declared/inherited writable source, never + // from a subtype — #248), so a scope over only those objects passed this + // refusal while governing zero tables. And not a fresh walk either: it would + // have to re-implement the builder's skip rules (abstract, TPH subtype, no + // writable source, `@unmanaged`) and would drift from them. + const fqns = [...new Set(provenance().values())]; + // A model that declares no table or view at all is not a scope error: there is + // nothing for a pattern to govern, scoped or not, and an empty schema has its + // own (much louder) failure modes downstream. + if (fqns.length === 0) return undefined; + if (fqns.some(inMigrateScope)) return undefined; + + const patterns = JSON.stringify(migrateScopePatterns ?? []); + const examples = fqns.slice(0, EXAMPLE_FQN_CAP).join(", "); + const more = fqns.length > EXAMPLE_FQN_CAP ? `, …and ${fqns.length - EXAMPLE_FQN_CAP} more` : ""; + return ( + `migrate.scope matched none of the ${fqns.length} object(s) declaring a table or view, ` + + `so this run would treat every one of them as another owner's and compare nothing. ` + + `Patterns: ${patterns}. Declaring a table or view: ${examples}${more}. ` + + `Fix the patterns in .metaobjects/config.json (migrate.scope), or remove the key to ` + + `govern everything the model declares.` + ); +} diff --git a/server/typescript/packages/cli/src/lib/output.ts b/server/typescript/packages/cli/src/lib/output.ts index 3bf9a3b27..c2382e605 100644 --- a/server/typescript/packages/cli/src/lib/output.ts +++ b/server/typescript/packages/cli/src/lib/output.ts @@ -192,7 +192,7 @@ export function genResultToData(result: GenResultShape): { ? `no entities to generate in ${result.outDir}` : parts.join(", "); const help = result.files.length === 0 - ? ["author entities under metaobjects/ then re-run `meta gen`"] + ? ["author entities in this project's metadata sources then re-run `meta gen`"] : ["typecheck the generated code with `npx tsc`", "create your database tables with `meta migrate --from-db --db --dialect --slug init --apply`"]; return { gen: result.files.map((f) => ({ file: f.path, status: f.status })), summary, help }; } diff --git a/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap b/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap index 6e6c6a66c..99fa1137c 100644 --- a/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap +++ b/server/typescript/packages/cli/test/__snapshots__/cli.test.ts.snap @@ -7,10 +7,10 @@ USAGE: meta [flags] COMMANDS: - init Scaffold metaobjects/ + .metaobjects/ in the current repo + init Scaffold a MetaObjects project in the current repo init --refresh-docs Refresh .metaobjects/AGENTS.md + CLAUDE.md after CLI upgrades agent-docs Scaffold only the agent-context (.metaobjects/ + .claude/skills/) — canonical redirect target for all language ports - gen [...] Codegen TS targets from metaobjects/ entities + gen [...] Codegen TS targets from your declared metadata types [query] Search the metadata vocabulary (types, subtypes, @attrs) by name or description export Flatten loaded metadata to one canonical JSON artifact docs --out Generate neutral metadata documentation (entity + template pages; --site for HTML site) @@ -33,7 +33,7 @@ EXPORT FLAGS: --out Write output to a file (default: stdout) DOCS FLAGS: - Project root holding metaobjects/ (default: current directory) + Project root to resolve metadata from (default: current directory) --out , -o Output directory for the pages (default: ./docs) --templates Project root to resolve adopter templates/ overrides (default: ) --prompts Extra dir holding prompt .mustache sources for --site (e.g. data/templates/) diff --git a/server/typescript/packages/cli/test/collection-routing.test.ts b/server/typescript/packages/cli/test/collection-routing.test.ts new file mode 100644 index 000000000..8810f72b5 --- /dev/null +++ b/server/typescript/packages/cli/test/collection-routing.test.ts @@ -0,0 +1,150 @@ +import { describe, test, expect, spyOn } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { genCommand } from "../src/commands/gen.js"; +import { run } from "../src/index.js"; + +// Place temp dirs inside the monorepo so metaobjects.config.ts's +// `@metaobjectsdev/*` imports resolve the same way the existing +// integration/gen-sqlite.test.ts fixtures do. +const WORKSPACE_TMP = resolve(import.meta.dirname, "fixtures/__tmp__"); + +function genOutDir(root: string): string { + return join(root, "generated", "db"); +} + +describe("gen routes metadata discovery through resolveCollection", () => { + test("generates from a sources-declared tree with no metaobjects/ present anywhere", async () => { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "collection-routing-")); + try { + mkdirSync(join(root, ".git")); + + // Metadata lives OUTSIDE the app directory entirely — under `model/`, + // not `metaobjects/` — and nowhere under `apps/ui`. + mkdirSync(join(root, "model"), { recursive: true }); + writeFileSync( + join(root, "model", "meta.a.json"), + JSON.stringify({ + "metadata.root": { + package: "acme", + children: [ + { + "object.entity": { + name: "Order", + children: [ + { "source.rdb": { "@table": "orders" } }, + { "field.long": { name: "id", "@column": "id" } }, + { + "identity.primary": { + name: "pk", + "@fields": ["id"], + "@generation": "increment", + }, + }, + ], + }, + }, + ], + }, + }), + ); + + // The app's config declares its own metadata source — a relative path + // outside the app dir — instead of relying on a `metaobjects/` default. + mkdirSync(join(root, "apps", "ui", ".metaobjects"), { recursive: true }); + writeFileSync( + join(root, "apps", "ui", ".metaobjects", "config.json"), + JSON.stringify({ + schema_version: 1, + sources: [{ path: "../../model" }], + }), + ); + + const appRoot = join(root, "apps", "ui"); + writeFileSync( + join(appRoot, "metaobjects.config.ts"), + ` +import { defineConfig } from "@metaobjectsdev/codegen-ts"; +import { entityFile } from "@metaobjectsdev/codegen-ts/generators"; +export default defineConfig({ + outDir: ${JSON.stringify(genOutDir(appRoot))}, + dialect: "sqlite", + dbImport: "~/db", + extStyle: "none", + generators: [entityFile()], +}); +`, + ); + + const code = await genCommand([], appRoot); + expect(code).toBe(0); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); + +// The probe in `run()` answers "is this a MetaObjects project?" through +// `resolveCollection`. What it PRINTS has to agree: a project whose config +// points `sources` at a sibling module has no directory of the default name at +// all, so "metaobjects/ found" is simply false — and a directory that resolves +// nothing may be failing on a declared source rather than on a missing default. +// Naming a directory in either message re-asserts the assumption the routing +// removed. +describe("the no-args project probe says what resolved, never a directory name", () => { + /** Run the CLI with no command and return everything it wrote to stdout. */ + async function statusOf(dir: string): Promise { + const lines: string[] = []; + const spy = spyOn(console, "log").mockImplementation((...args: unknown[]) => { + lines.push(args.map(String).join(" ")); + }); + try { + expect(await run(["--cwd", dir])).toBe(0); + } finally { + spy.mockRestore(); + } + return lines.join("\n"); + } + + test("a project whose sources point elsewhere is not told a directory was found", async () => { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "probe-declared-")); + try { + mkdirSync(join(root, ".git")); + mkdirSync(join(root, "model"), { recursive: true }); + writeFileSync( + join(root, "model", "meta.a.json"), + JSON.stringify({ + "metadata.root": { package: "acme", children: [{ "object.entity": { name: "Order" } }] }, + }), + ); + mkdirSync(join(root, "apps", "ui", ".metaobjects"), { recursive: true }); + writeFileSync( + join(root, "apps", "ui", ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, sources: [{ path: "../../model" }] }), + ); + + const out = await statusOf(join(root, "apps", "ui")); + expect(out).toContain("metadata found"); + expect(out).not.toContain("metaobjects/"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("a directory with no project names no directory either", async () => { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "probe-empty-")); + try { + mkdirSync(join(root, ".git")); + const out = await statusOf(root); + expect(out).toContain("no MetaObjects project here"); + // Including the next-step line: `meta init` scaffolds a project, and the + // layout it writes is its own business to describe, not this probe's. + expect(out).not.toContain("metaobjects/"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/cli/test/docs-command.test.ts b/server/typescript/packages/cli/test/docs-command.test.ts index fcd1ea756..026e7a004 100644 --- a/server/typescript/packages/cli/test/docs-command.test.ts +++ b/server/typescript/packages/cli/test/docs-command.test.ts @@ -37,6 +37,21 @@ const META = { }, }; +/** A second model in its own package, for the shared-source `--site` case. */ +const SHARED_META = { + "metadata.root": { + package: "acme::shared", + children: [ + { + "object.value": { + name: "SharedThing", + children: [{ "field.string": { name: "label" } }], + }, + }, + ], + }, +}; + const dirs: string[] = []; /** Build a standalone project root holding metaobjects/ — NO gen config. Also @@ -450,6 +465,41 @@ describe("meta docs --site — HTML documentation site", () => { expect(code).toBe(0); expect(existsSync(join(out, "site", "index.html"))).toBe(true); }); + + test("two declared sources sharing a basename are disambiguated, not refused", async () => { + // The flagship shape of the multi-source feature: a project's own + // `metaobjects/` plus a shared model's `metaobjects/` next door. Both have + // the basename the site keys its source groups by, and the site used to + // refuse the pair outright ("duplicate source dir basename"). + const workspace = await mkdtemp(join(tmpdir(), "meta-docs-multi-")); + dirs.push(workspace); + const root = join(workspace, "app"); + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await mkdir(join(root, "metaobjects"), { recursive: true }); + await writeFile(join(root, "metaobjects", "meta.json"), JSON.stringify(META), "utf8"); + await mkdir(join(workspace, "shared-model", "metaobjects"), { recursive: true }); + await writeFile( + join(workspace, "shared-model", "metaobjects", "meta.json"), + JSON.stringify(SHARED_META), + "utf8", + ); + await writeFile( + join(root, ".metaobjects", "config.json"), + JSON.stringify({ + schema_version: 1, + sources: [{ path: "metaobjects" }, { path: "../shared-model/metaobjects" }], + }), + "utf8", + ); + + const out = join(root, "out-site-multi"); + expect(await docsCommand([root, "--site", "--out", out], root)).toBe(0); + expect(existsSync(join(out, "site", "index.html"))).toBe(true); + // Both models are in the site, so neither source was dropped to dodge the collision. + const index = await readFile(join(out, "site", "index.html"), "utf8"); + expect(index).toContain("Welcome"); + expect(index).toContain("SharedThing"); + }); }); describe("meta docs --scaffold-site — own your theme", () => { diff --git a/server/typescript/packages/cli/test/gen-split-tree-single-import.test.ts b/server/typescript/packages/cli/test/gen-split-tree-single-import.test.ts index 845f35140..04785deb5 100644 --- a/server/typescript/packages/cli/test/gen-split-tree-single-import.test.ts +++ b/server/typescript/packages/cli/test/gen-split-tree-single-import.test.ts @@ -99,9 +99,13 @@ function ensureFreshDist(): void { const codegenTsRoot = dirname( createRequire(import.meta.url).resolve("@metaobjectsdev/codegen-ts/package.json"), ); + const sdkRoot = dirname( + createRequire(import.meta.url).resolve("@metaobjectsdev/sdk/package.json"), + ); for (const { name, pkgRoot, srcDir, distFile } of [ { name: "codegen-ts", pkgRoot: codegenTsRoot, srcDir: join(codegenTsRoot, "src"), distFile: join(codegenTsRoot, "dist", "index.js") }, { name: "cli", pkgRoot: CLI_ROOT, srcDir: join(CLI_ROOT, "src"), distFile: META_BIN }, + { name: "sdk", pkgRoot: sdkRoot, srcDir: join(sdkRoot, "src"), distFile: join(sdkRoot, "dist", "index.js") }, ]) { const stale = (): boolean => !existsSync(distFile) || newestSrcMtime(srcDir) > statSync(distFile).mtimeMs; diff --git a/server/typescript/packages/cli/test/init.test.ts b/server/typescript/packages/cli/test/init.test.ts index 9a0d559ee..11ce1f68f 100644 --- a/server/typescript/packages/cli/test/init.test.ts +++ b/server/typescript/packages/cli/test/init.test.ts @@ -152,7 +152,7 @@ describe("init() --force config preservation", () => { schema_version: 1 as const, pending_in_git: false, // changed from default confidence_thresholds: { pending_promote: 0.95, drift_warn: 0.8 }, - sources: [{ kind: "package" as const, package: "@acme/entities" }], + sources: [{ package: "@acme/entities" }], extract: {}, }; await saveConfig(join(cwd, ".metaobjects"), ConfigSchema.parse(customConfig)); @@ -165,7 +165,7 @@ describe("init() --force config preservation", () => { const reloaded = JSON.parse(readFileSync(join(cwd, ".metaobjects", "config.json"), "utf8")); expect(reloaded.pending_in_git).toBe(false); expect(reloaded.confidence_thresholds.pending_promote).toBe(0.95); - expect(reloaded.sources).toEqual([{ kind: "package", package: "@acme/entities" }]); + expect(reloaded.sources).toEqual([{ package: "@acme/entities" }]); }); test("writes fresh defaults when existing config is invalid (and warns)", async () => { diff --git a/server/typescript/packages/cli/test/integration/gen-scope.test.ts b/server/typescript/packages/cli/test/integration/gen-scope.test.ts new file mode 100644 index 000000000..b2fc1f15c --- /dev/null +++ b/server/typescript/packages/cli/test/integration/gen-scope.test.ts @@ -0,0 +1,105 @@ +/** + * Task 12b: `meta gen` honours the collection-level `scope` declared in + * `.metaobjects/config.json`, filtering GENERATED output — never input. The + * collection still loads the whole model; only the emitted file set narrows. + */ +import { describe, test, expect } from "bun:test"; +import { cpSync, mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { run } from "../../src/index.js"; + +const FIXTURES = resolve(import.meta.dirname, "../fixtures"); +// Place temp dirs inside the monorepo so jiti can resolve @metaobjectsdev/* +// when it loads metaobjects.config.ts (same rationale as gen-sqlite.test.ts). +const WORKSPACE_TMP = resolve(import.meta.dirname, "../fixtures/__tmp__"); + +function genOutDir(root: string): string { + return join(root, "generated", "db"); +} + +/** trainer-website-meta declares User/Post/Tag, all in package "trainerWebsite". */ +function setupRepo(): string { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "forge-gen-scope-")); + cpSync(join(FIXTURES, "trainer-website-meta"), root, { recursive: true }); + writeFileSync( + join(root, "metaobjects.config.ts"), + ` +import { defineConfig } from "@metaobjectsdev/codegen-ts"; +import { entityFile } from "@metaobjectsdev/codegen-ts/generators"; +export default defineConfig({ + outDir: ${JSON.stringify(genOutDir(root))}, + dialect: "sqlite", + dbImport: "~/db", + extStyle: "none", + generators: [entityFile()], +}); +`, + ); + return root; +} + +function declareScope(repo: string, include: string[]): void { + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, scope: { include } }), + "utf8", + ); +} + +describe("meta gen — collection scope", () => { + test("a declared scope emits only the in-scope entity's files", async () => { + const root = setupRepo(); + try { + declareScope(root, ["trainerWebsite::Post"]); + + const exit = await run(["gen", "--cwd", root]); + expect(exit).toBe(0); + + const outDir = genOutDir(root); + const files = readdirSync(outDir); + expect(files).toContain("Post.ts"); + expect(files).not.toContain("User.ts"); + expect(files).not.toContain("Tag.ts"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + + test("an unscoped run emits every entity, byte-for-byte the same as an all-matching scope", async () => { + // Previously titled "byte-identical to today" while asserting only that three + // FILENAMES were present. A title claiming a byte guarantee over a test that + // reads no bytes is how the real guarantee went unexamined — the resolver was + // reordering the loaded file list, and therefore the emitted content, with + // nothing here able to see it. + const unscoped = setupRepo(); + const allMatching = setupRepo(); + try { + // No .metaobjects/config.json at all — the default, unscoped path. + expect(await run(["gen", "--cwd", unscoped])).toBe(0); + // A scope that admits everything must be indistinguishable from no scope. + declareScope(allMatching, ["trainerWebsite::**"]); + expect(await run(["gen", "--cwd", allMatching])).toBe(0); + + const names = readdirSync(genOutDir(unscoped)).sort(); + expect(names).toContain("Post.ts"); + expect(names).toContain("User.ts"); + expect(names).toContain("Tag.ts"); + expect(readdirSync(genOutDir(allMatching)).sort()).toEqual(names); + + for (const name of names) { + const a = readFileSync(join(genOutDir(unscoped), name), "utf8"); + const b = readFileSync(join(genOutDir(allMatching), name), "utf8"); + // Real bytes, not a filename listing. The outDir is baked into each + // repo's config, so nothing generated should mention it — if that ever + // changes, this is the assertion that says so. + expect(a).not.toContain(unscoped); + expect(b).toBe(a); + } + } finally { + rmSync(unscoped, { recursive: true, force: true }); + rmSync(allMatching, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/cli/test/integration/migrate-config-dir.test.ts b/server/typescript/packages/cli/test/integration/migrate-config-dir.test.ts new file mode 100644 index 000000000..3c03df7c9 --- /dev/null +++ b/server/typescript/packages/cli/test/integration/migrate-config-dir.test.ts @@ -0,0 +1,128 @@ +/** + * The config a command reads comes from the directory the METADATA was resolved + * from, never from ambient cwd. + * + * `metaobjects.config.ts` carries `columnNamingStrategy`. Once metadata resolves + * from the nearest ancestor holding `.metaobjects/config.json`, reading that file + * from cwd instead silently splits the two: run `meta migrate` from a subdirectory + * of a project whose root declares `literal` and the metadata comes from the + * ancestor while the strategy defaults to `snake_case` — emitting a migration that + * RENAMES EVERY COLUMN. Newly reachable, too: before metadata sources were + * resolvable, that invocation just failed with "no metaobjects/ found". + * + * The gate is byte-level and comparative, not a spot-check on one identifier: the + * SQL a subdirectory run emits must be byte-identical to the SQL the project-root + * run emits. A drifting default shows up as a diff whether or not anyone thought + * to assert on the setting that drifted. + */ +import { describe, test, expect } from "bun:test"; +import { mkdtempSync, mkdirSync, rmSync, readdirSync, readFileSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { run } from "../../src/index.js"; + +// Temp dirs live inside the monorepo so jiti can resolve @metaobjectsdev/* when +// it loads metaobjects.config.ts (same rationale as gen-sqlite.test.ts). +const WORKSPACE_TMP = resolve(import.meta.dirname, "../fixtures/__tmp__"); + +const USERS = JSON.stringify({ + "metadata.root": { + package: "acme", + children: [{ + "object.entity": { + name: "User", + children: [ + { "source.rdb": { name: "src", "@table": "users" } }, + { "field.long": { name: "id" } }, + // Two words, so `literal` and `snake_case` produce DIFFERENT column names. + { "field.string": { name: "firstName" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +/** A project whose ROOT declares `columnNamingStrategy: "literal"`, with an + * otherwise-empty subdirectory to run from. */ +function scaffold(): string { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const repo = mkdtempSync(join(WORKSPACE_TMP, "migrate-config-dir-")); + mkdirSync(join(repo, "metaobjects"), { recursive: true }); + writeFileSync(join(repo, "metaobjects", "meta.users.json"), USERS, "utf8"); + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1 }), + "utf8", + ); + writeFileSync( + join(repo, "metaobjects.config.ts"), + ` +import { defineConfig } from "@metaobjectsdev/codegen-ts"; +export default defineConfig({ + outDir: ${JSON.stringify(join(repo, "generated"))}, + dialect: "sqlite", + columnNamingStrategy: "literal", + generators: [], +}); +`, + "utf8", + ); + mkdirSync(join(repo, "apps", "api"), { recursive: true }); + return repo; +} + +/** The `up.sql` the run wrote, read out of the project root's migrations dir. */ +function emittedUpSql(repo: string): string { + const migrations = join(repo, ".metaobjects", "migrations"); + const dirs = readdirSync(migrations).filter((d) => d.endsWith("-init")); + expect(dirs).toHaveLength(1); + return readFileSync(join(migrations, dirs[0]!, "up.sql"), "utf8"); +} + +async function migrateFrom(repo: string, runDir: string): Promise { + const exit = await run([ + "migrate", "--from-db", "--cwd", runDir, + "--db", `file:${join(repo, "local.db")}`, + "--dialect", "sqlite", "--slug", "init", + ]); + expect(exit).toBe(0); + return emittedUpSql(repo); +} + +describe("meta migrate — config comes from the resolved config dir", () => { + test("a subdirectory run emits byte-identical SQL to a project-root run", async () => { + const fromRoot = scaffold(); + const fromSubdir = scaffold(); + try { + const rootSql = await migrateFrom(fromRoot, fromRoot); + const subdirSql = await migrateFrom(fromSubdir, join(fromSubdir, "apps", "api")); + + // Both runs honour the root's `literal` strategy. Asserted explicitly as + // well as comparatively, so a failure says WHICH way it went rather than + // only that the two disagree. + expect(rootSql).toContain("firstName"); + expect(rootSql).not.toContain("first_name"); + + // The paths differ per temp dir, so compare the SQL bodies only — nothing + // in generated DDL should mention an absolute path anyway. + expect(subdirSql).toBe(rootSql); + } finally { + rmSync(fromRoot, { recursive: true, force: true }); + rmSync(fromSubdir, { recursive: true, force: true }); + } + }); + + test("a subdirectory run writes its migration under the project root, not the subdirectory", async () => { + const repo = scaffold(); + try { + await migrateFrom(repo, join(repo, "apps", "api")); + // `outDir` is relative (`./.metaobjects/migrations`) and must resolve against + // the config dir, or the migration lands somewhere the next run cannot find. + expect(readdirSync(join(repo, ".metaobjects", "migrations")).length).toBeGreaterThan(0); + expect(readdirSync(join(repo, "apps", "api"))).toEqual([]); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts new file mode 100644 index 000000000..aac031040 --- /dev/null +++ b/server/typescript/packages/cli/test/integration/migrate-db-scope.test.ts @@ -0,0 +1,132 @@ +/** + * `meta migrate --db` (the ONLINE, live-introspection path) and `migrate.scope`. + * + * The scope feature shipped with no test on this path at all — every existing scope + * test drives either the offline diff or `verify --db`. It is also the path where a + * wrong scope is most expensive: it introspects a real database and writes DDL. + * + * The case under test is the one that inverts: a scope matching NOTHING. It is + * always an authoring error (a typo'd or stale package pattern), it can never be + * what someone meant, and left alone it is silent — migrate reports "no changes" + * having compared nothing, while an empty expected side is exactly what the diff + * reads as "no model, govern the whole database". Refused, with the patterns and + * the loaded FQNs named, so the author can see what missed. + * + * The near-miss variant matters just as much: a scope matching only value + * objects and abstracts matches LOADED objects but none that can declare a + * table or view — the run still compares nothing, so it is refused on the same + * question, answered against the expected schema's provenance rather than the + * loaded object set. + */ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { rmSync } from "node:fs"; +import { run } from "../../src/index.js"; +import { declareScope, scaffold } from "./support/scope-fixture.js"; + +const migrateFromDb = (repo: string, dbUrl: string): Promise => + run(["migrate", "--from-db", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite", "--slug", "initial"]); + +let out: string[]; +let err: string[]; +let origLog: typeof console.log; +let origErr: typeof console.error; + +beforeEach(() => { + out = []; + err = []; + origLog = console.log; + origErr = console.error; + console.log = (...a: unknown[]) => { out.push(a.map(String).join(" ")); }; + console.error = (...a: unknown[]) => { err.push(a.map(String).join(" ")); }; +}); +afterEach(() => { + console.log = origLog; + console.error = origErr; +}); + +describe("meta migrate --db — migrate.scope", () => { + test("a scope matching NO loaded object is refused, naming the patterns and what was loaded", async () => { + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); + try { + declareScope(repo, ["typo::**"]); + expect(await migrateFromDb(repo, dbUrl)).toBe(2); + const all = [...out, ...err].join("\n"); + expect(all).toContain("matched none"); + // The patterns that missed, and the shape they had to match — an author + // cannot fix a typo from "your scope matched nothing" alone. + expect(all).toContain("typo::**"); + expect(all).toContain("acme::platform::Job"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("a scope matching only value objects and abstracts is refused — they declare no table", async () => { + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); + try { + // `acme::shared` (scaffolded by the fixture) holds an abstract base and a + // value object: loaded objects, but none that can contribute a table or + // view. Matching them is not governing anything — the run would compare + // nothing and report "no changes" against a database it was told to check. + declareScope(repo, ["acme::shared::**"]); + expect(await migrateFromDb(repo, dbUrl)).toBe(2); + const all = [...out, ...err].join("\n"); + expect(all).toContain("matched none"); + // The patterns that missed, and the table-declaring objects they could + // have matched — the refusal is decided against those, not against every + // loaded object. + expect(all).toContain("acme::shared::**"); + expect(all).toContain("acme::platform::Job"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("a scope that matches something still runs (the refusal is not a blanket break)", async () => { + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); + try { + declareScope(repo, ["acme::platform::**"]); + expect(await migrateFromDb(repo, dbUrl)).toBe(0); + const all = [...out, ...err].join("\n"); + expect(all).not.toContain("matched none"); + // `matches` belongs to the other owner: reported as out-of-scope, never created. + expect(all).toContain("out-of-scope"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("--format json stays parseable under a scope — the out-of-scope note is text-format only", async () => { + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); + try { + declareScope(repo, ["acme::platform::**"]); + expect(await run([ + "--format", "json", "migrate", "--from-db", "--cwd", repo, + "--db", dbUrl, "--dialect", "sqlite", "--slug", "initial", + ])).toBe(0); + // The whole point: stdout is ONE machine-readable document. A prose line + // ahead of it breaks `| jq` outright, which is how the out-of-scope note + // shipped — unconditional `log.info`. + const stdout = out.join("\n"); + expect(stdout).not.toContain("out-of-scope"); + expect(() => JSON.parse(stdout)).not.toThrow(); + // Moved to stderr, not dropped — an object that was neither created nor + // dropped has to be reported somewhere, in every format. + expect(err.join("\n")).toContain("out-of-scope"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("no migrate.scope declared — unchanged, both tables governed", async () => { + const { repo, dbUrl } = scaffold("metaobjects-migrate-scope-"); + try { + expect(await migrateFromDb(repo, dbUrl)).toBe(0); + const all = [...out, ...err].join("\n"); + expect(all).not.toContain("matched none"); + expect(all).not.toContain("out-of-scope"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/cli/test/integration/support/scope-fixture.ts b/server/typescript/packages/cli/test/integration/support/scope-fixture.ts new file mode 100644 index 000000000..4966c363e --- /dev/null +++ b/server/typescript/packages/cli/test/integration/support/scope-fixture.ts @@ -0,0 +1,113 @@ +/** + * The two-owner project every `migrate.scope` integration test drives. + * + * `meta migrate` and `meta verify --db` govern the identical object set from ONE + * declaration, so their integration tests scaffold the identical project — and + * two copies of it had already drifted (`ARENA` was a constant in one file and a + * `(venue: boolean)` factory in the other), which is how two tests that are + * supposed to prove the same contract quietly stop testing the same thing. + * + * The shape: this consumer's `acme::platform` package owns `jobs`; a second + * owner's `arena` package owns `matches` in the same database. A scope of + * `["acme::platform::**"]` therefore governs exactly one of the two. + */ +import { mkdtempSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** This consumer's package. */ +export const PLATFORM = JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [{ + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +/** + * Another owner's package, sharing the database. + * + * `venue` models a column the other owner has declared but not migrated yet — + * drift for THEM, never for this consumer. Pass `false` for the base shape. + */ +export const arena = (opts: { venue: boolean } = { venue: false }): string => JSON.stringify({ + "metadata.root": { + package: "arena", + children: [{ + "object.entity": { + name: "Match", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + ...(opts.venue ? [{ "field.string": { name: "venue" } }] : []), + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }], + }, +}); + +/** Absolute path of the arena metadata file, for a test that rewrites it. */ +export const arenaFile = (repo: string): string => join(repo, "metaobjects", "meta.arena.json"); + +/** + * A package of shared SHAPES: an abstract base and a value object. Both are + * loaded objects, but neither can declare a table or view — persistability + * needs a writable source — so a `migrate.scope` over only this package + * governs zero tables however well its patterns match. + */ +export const SHARED = JSON.stringify({ + "metadata.root": { + package: "acme::shared", + children: [ + { + "object.entity": { + name: "BaseRecord", + abstract: true, + children: [{ "field.long": { name: "id" } }], + }, + }, + { + "object.value": { + name: "Address", + children: [ + { "field.string": { name: "line1" } }, + { "field.string": { name: "line2" } }, + ], + }, + }, + ], + }, +}); + +/** + * A throwaway project holding both packages, plus the sqlite URL beside it. + * `prefix` names the temp directory so a failing run says which suite made it. + */ +export function scaffold(prefix: string): { repo: string; dbUrl: string } { + const repo = mkdtempSync(join(tmpdir(), prefix)); + mkdirSync(join(repo, "metaobjects"), { recursive: true }); + writeFileSync(join(repo, "metaobjects", "meta.platform.json"), PLATFORM, "utf8"); + writeFileSync(arenaFile(repo), arena(), "utf8"); + writeFileSync(join(repo, "metaobjects", "meta.shared.json"), SHARED, "utf8"); + return { repo, dbUrl: `file:${join(repo, "local.db")}` }; +} + +/** Declare `migrate.scope` on an existing scaffold — the ONE key both commands read. */ +export function declareScope(repo: string, scope: string[]): void { + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { scope } }), + "utf8", + ); +} diff --git a/server/typescript/packages/cli/test/integration/verify-codegen-scope.test.ts b/server/typescript/packages/cli/test/integration/verify-codegen-scope.test.ts new file mode 100644 index 000000000..fba8ecb76 --- /dev/null +++ b/server/typescript/packages/cli/test/integration/verify-codegen-scope.test.ts @@ -0,0 +1,91 @@ +/** + * Task 12b, requirement 3 / design §7 open question 3: `verify --codegen` must + * regenerate under the SAME `collection.scope` `meta gen` used to produce the + * committed output — otherwise every out-of-scope entity reads as drift (regen + * would try to emit it; it was never committed because `meta gen` never emitted + * it either). + */ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { cpSync, mkdtempSync, mkdirSync, rmSync, writeFileSync } from "node:fs"; +import { join, resolve } from "node:path"; +import { run } from "../../src/index.js"; + +const FIXTURES = resolve(import.meta.dirname, "../fixtures"); +const WORKSPACE_TMP = resolve(import.meta.dirname, "../fixtures/__tmp__"); + +function genOutDir(root: string): string { + return join(root, "generated", "db"); +} + +/** trainer-website-meta declares User/Post/Tag, all in package "trainerWebsite". */ +function setupRepo(): string { + mkdirSync(WORKSPACE_TMP, { recursive: true }); + const root = mkdtempSync(join(WORKSPACE_TMP, "forge-verify-codegen-scope-")); + cpSync(join(FIXTURES, "trainer-website-meta"), root, { recursive: true }); + writeFileSync( + join(root, "metaobjects.config.ts"), + ` +import { defineConfig } from "@metaobjectsdev/codegen-ts"; +import { entityFile } from "@metaobjectsdev/codegen-ts/generators"; +export default defineConfig({ + outDir: ${JSON.stringify(genOutDir(root))}, + dialect: "sqlite", + dbImport: "~/db", + extStyle: "none", + generators: [entityFile()], +}); +`, + ); + return root; +} + +function declareScope(repo: string, include: string[]): void { + mkdirSync(join(repo, ".metaobjects"), { recursive: true }); + writeFileSync( + join(repo, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, scope: { include } }), + "utf8", + ); +} + +let out: string[]; +let err: string[]; +let origLog: typeof console.log; +let origErr: typeof console.error; + +beforeEach(() => { + out = []; + err = []; + origLog = console.log; + origErr = console.error; + console.log = (...a: unknown[]) => { out.push(a.map(String).join(" ")); }; + console.error = (...a: unknown[]) => { err.push(a.map(String).join(" ")); }; +}); +afterEach(() => { + console.log = origLog; + console.error = origErr; +}); + +describe("meta verify --codegen — collection scope", () => { + test("reports no drift for out-of-scope entities the scoped gen never committed", async () => { + const root = setupRepo(); + try { + declareScope(root, ["trainerWebsite::Post"]); + + // `meta gen` under the scope commits ONLY Post.ts. + expect(await run(["gen", "--cwd", root])).toBe(0); + + // `verify --codegen` must regenerate under the identical scope — if it + // regenerated unscoped, User.ts/Tag.ts would appear in the fresh tree + // but not the committed one, reading as drift on entities this scope + // deliberately excludes. + const exit = await run(["verify", "--cwd", root, "--codegen"]); + const all = [...out, ...err].join("\n"); + expect(exit).toBe(0); + expect(all).not.toContain("User.ts"); + expect(all).not.toContain("Tag.ts"); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts b/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts new file mode 100644 index 000000000..215169e97 --- /dev/null +++ b/server/typescript/packages/cli/test/integration/verify-db-scope.test.ts @@ -0,0 +1,112 @@ +/** + * `meta verify --db` honours `migrate.scope` (real sqlite, whole CLI pipeline). + * + * `migrate` and `verify --db` govern the identical object set — a drift gate + * that fails on tables `migrate` deliberately does not own is incoherent — so + * the two share ONE declaration (`migrate.scope`) rather than a second key. + * An out-of-scope object is reported as out-of-scope, never as drift: silence + * alone would misreport an unchecked table as a checked one. + */ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { rmSync, writeFileSync, readFileSync, readdirSync } from "node:fs"; +import { join } from "node:path"; +import { createClient } from "@libsql/client"; +import { run } from "../../src/index.js"; +import { arena, arenaFile, declareScope, PLATFORM, scaffold } from "./support/scope-fixture.js"; + +/** Materialize the current metadata schema into the DB via the real migrate path. */ +async function materialize(repo: string, dbUrl: string): Promise { + const exit = await run(["migrate", "--from-db", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite", "--slug", "initial"]); + expect(exit).toBe(0); + const migrationsRoot = join(repo, ".metaobjects", "migrations"); + const dir = readdirSync(migrationsRoot).find((s) => s.endsWith("-initial"))!; + const sql = readFileSync(join(migrationsRoot, dir, "up.sql"), "utf8"); + const client = createClient({ url: dbUrl }); + for (const stmt of sql.split(";").map((s) => s.trim()).filter((s) => s.length > 0)) { + await client.execute(stmt); + } + client.close(); +} + +let out: string[]; +let err: string[]; +let origLog: typeof console.log; +let origErr: typeof console.error; + +beforeEach(() => { + out = []; + err = []; + origLog = console.log; + origErr = console.error; + console.log = (...a: unknown[]) => { out.push(a.map(String).join(" ")); }; + console.error = (...a: unknown[]) => { err.push(a.map(String).join(" ")); }; +}); +afterEach(() => { + console.log = origLog; + console.error = origErr; +}); + +describe("meta verify --db — migrate.scope", () => { + test("an out-of-scope object's divergence is reported as out-of-scope, not as drift", async () => { + const { repo, dbUrl } = scaffold("metaobjects-verify-scope-"); + try { + await materialize(repo, dbUrl); + expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(0); + + declareScope(repo, ["acme::platform::**"]); + // The other owner's model gains a column its own migration has not applied. + writeFileSync(arenaFile(repo), arena({ venue: true }), "utf8"); + out = []; + err = []; + + expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(0); + const all = [...out, ...err].join("\n"); + expect(all).toContain("out-of-scope"); + expect(all).toContain("matches"); + expect(all).not.toContain("venue"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("in-scope drift still fails the gate under a scope", async () => { + const { repo, dbUrl } = scaffold("metaobjects-verify-scope-"); + try { + await materialize(repo, dbUrl); + declareScope(repo, ["acme::platform::**"]); + // This consumer's OWN model gains a column the database lacks. + writeFileSync( + join(repo, "metaobjects", "meta.platform.json"), + PLATFORM.replace( + `{"field.string":{"name":"title"}}`, + `{"field.string":{"name":"title"}},{"field.string":{"name":"owner"}}`, + ), + "utf8", + ); + out = []; + err = []; + + expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(1); + expect([...out, ...err].join("\n")).toContain("owner"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); + + test("with no migrate.scope declared, the same divergence IS drift (unchanged)", async () => { + const { repo, dbUrl } = scaffold("metaobjects-verify-scope-"); + try { + await materialize(repo, dbUrl); + writeFileSync(arenaFile(repo), arena({ venue: true }), "utf8"); + out = []; + err = []; + + expect(await run(["verify", "--cwd", repo, "--db", dbUrl, "--dialect", "sqlite"])).toBe(1); + const all = [...out, ...err].join("\n"); + expect(all).toContain("venue"); + expect(all).not.toContain("out-of-scope"); + } finally { + rmSync(repo, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/cli/test/migrate-format-flyway.test.ts b/server/typescript/packages/cli/test/migrate-format-flyway.test.ts index 36819cc38..8007bbbb3 100644 --- a/server/typescript/packages/cli/test/migrate-format-flyway.test.ts +++ b/server/typescript/packages/cli/test/migrate-format-flyway.test.ts @@ -4,7 +4,7 @@ // generate but never apply), and the emit layout (V__/U__ into Flyway's // conventional dir, with --out-dir overriding it). -import { describe, test, expect, afterAll } from "bun:test"; +import { describe, test, expect, afterAll, spyOn } from "bun:test"; import { mkdtemp, rm, mkdir, writeFile, readdir } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -195,3 +195,78 @@ describe("migrate --migration-format flyway — emit", () => { expect(entries[0]!.endsWith("-add-note")).toBe(true); }); }); + +describe("the relocated-ledger warning under the flyway layout", () => { + // The warning exists to say "the ledger you can see here is not the one this + // run uses". Under `--migration-format flyway` with a default `outDir` the + // directory the run uses comes from `resolveFormatOutDir`, which redirects to + // Flyway's conventional location — so comparing against the unredirected + // `outDir` named a directory the run would never touch. + test("names the directory the run will actually use, not the default outDir", async () => { + const root = await mkdtemp(join(tmpdir(), "mts-flyway-warn-")); + dirs.push(root); + // The project root declares the config; the subdirectory the command runs + // from holds a ledger of its own but no config — the exact layout the + // warning was written for. + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await writeFile(join(root, ".metaobjects", "config.json"), '{"schema_version":1}', "utf8"); + const sub = join(root, "apps", "api"); + await mkdir(join(sub, ".metaobjects", "migrations"), { recursive: true }); + + const stderr: string[] = []; + const spy = spyOn(console, "error").mockImplementation((...args: unknown[]) => { + stderr.push(args.map(String).join(" ")); + }); + try { + // `--dialect d1` is refused by the flyway adapter immediately AFTER the + // warning, so this exercises the warning without needing a database. + await run(["migrate", "--cwd", sub, "--migration-format", "flyway", "--dialect", "d1"]); + } finally { + spy.mockRestore(); + } + + const warning = stderr.find((l) => l.includes("using the migrations directory")); + expect(warning).toBeDefined(); + expect(warning).toContain(join(root, "src", "main", "resources", "db", "migration")); + expect(warning).not.toContain(join(root, ".metaobjects", "migrations")); + }); +}); + +describe("the relocated-ledger warning under a plain d1 run", () => { + // A default (non-flyway) `--dialect d1` run has its OWN directory + // convention — wrangler.toml's `migrations_dir`, falling back to + // `"migrations"` — which the Kysely-path `resolveFormatOutDir` knows + // nothing about. Before the fix, this case named the Kysely-path default + // (`.metaobjects/migrations`), a directory a d1 run never writes to. + test("names d1's own migrations directory, not the Kysely-path default", async () => { + const root = await mkdtemp(join(tmpdir(), "mts-d1-warn-")); + dirs.push(root); + // The project root declares the config; the subdirectory the command runs + // from holds a ledger of its own but no config — the exact layout the + // warning was written for. + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await writeFile(join(root, ".metaobjects", "config.json"), '{"schema_version":1}', "utf8"); + const sub = join(root, "apps", "api"); + await mkdir(join(sub, ".metaobjects", "migrations"), { recursive: true }); + + const stderr: string[] = []; + const spy = spyOn(console, "error").mockImplementation((...args: unknown[]) => { + stderr.push(args.map(String).join(" ")); + }); + try { + // No wrangler.toml and no `metaobjects/` directory: an explicit --d1 + // binding bypasses the wrangler.toml requirement, so the run reaches + // the warning (issued right after the binding resolves) and then fails + // cleanly at metadata resolution (ERR_COLLECTION_NOT_FOUND, exit 2) — + // exercising the warning without needing a real D1 database. + await run(["migrate", "--cwd", sub, "--dialect", "d1", "--d1", "DB"]); + } finally { + spy.mockRestore(); + } + + const warning = stderr.find((l) => l.includes("using the migrations directory")); + expect(warning).toBeDefined(); + expect(warning).toContain(join(root, "migrations")); + expect(warning).not.toContain(join(root, ".metaobjects", "migrations")); + }); +}); diff --git a/server/typescript/packages/cli/test/migrate-scope.test.ts b/server/typescript/packages/cli/test/migrate-scope.test.ts new file mode 100644 index 000000000..4c6f27bb7 --- /dev/null +++ b/server/typescript/packages/cli/test/migrate-scope.test.ts @@ -0,0 +1,181 @@ +/** + * `migrate.scope` — a `meta migrate` run governs only the objects it declares. + * + * Without this, "load everything" turns a real adopter's worst standing hazard — + * a migrate proposing to DROP tables it does not model — from a discipline into + * an automation. The suppression is BOTH-sided: the out-of-scope tables leave + * the expected schema AND are excluded from the actual side, so the run neither + * creates nor drops them. + * + * Boundary (deliberately unchanged): a table that NO loaded object declares is + * still a proposed drop. Scope only silences tables whose declaring object was + * loaded and fell outside it. + */ +import { describe, test, expect, afterAll } from "bun:test"; +import { mkdtemp, rm, mkdir, writeFile, readdir, readFile, unlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runBaseline, runOfflineGenerate } from "../src/commands/migrate.js"; + +const dirs: string[] = []; +afterAll(async () => { for (const d of dirs) await rm(d, { recursive: true, force: true }); }); + +const PLATFORM = (extraField: boolean): string => JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [{ + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title", "@maxLength": 80 } }, + ...(extraField ? [{ "field.string": { name: "note" } }] : []), + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }], + }, +}); + +/** Another owner's package, in the same database. */ +const ARENA = (extraField: boolean): string => JSON.stringify({ + "metadata.root": { + package: "arena", + children: [{ + "object.entity": { + name: "Match", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + ...(extraField ? [{ "field.string": { name: "venue" } }] : []), + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }], + }, +}); + +async function project(): Promise { + const root = await mkdtemp(join(tmpdir(), "migrate-scope-")); + dirs.push(root); + await mkdir(join(root, "metaobjects"), { recursive: true }); + await writeFile(join(root, "metaobjects", "meta.platform.json"), PLATFORM(false), "utf8"); + await writeFile(join(root, "metaobjects", "meta.arena.json"), ARENA(false), "utf8"); + return root; +} + +async function declareScope(root: string, scope: string[]): Promise { + await mkdir(join(root, ".metaobjects"), { recursive: true }); + await writeFile( + join(root, ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, migrate: { scope } }), + "utf8", + ); +} + +const cfg = () => + ({ dialect: "sqlite", outDir: "./.metaobjects/migrations", onAmbiguous: "abort", + allow: [], slug: "auto", dryRun: false } as never); + +const migrationDirs = async (root: string): Promise => + (await readdir(join(root, ".metaobjects/migrations"))).filter((e) => !e.startsWith(".")); + +describe("meta migrate — migrate.scope", () => { + test("an out-of-scope table is neither altered nor dropped", async () => { + const root = await project(); + // Baseline BEFORE the scope is declared: the reference snapshot records both + // owners' tables, exactly as a `--from-db` baseline of the shared database would. + expect(await runBaseline(cfg(), root)).toBe(0); + await declareScope(root, ["acme::platform::**"]); + // The other owner evolves ITS model. Three outcomes are distinguishable here: + // no scoping at all migrates the foreign column; scoping the EXPECTED side alone + // proposes DROP TABLE "matches" (blocked → exit 1); correct both-sided + // suppression produces silence. + await writeFile(join(root, "metaobjects", "meta.arena.json"), ARENA(true), "utf8"); + + expect(await runOfflineGenerate(cfg(), root)).toBe(0); + expect(await migrationDirs(root)).toHaveLength(0); + }); + + test("in-scope changes still migrate under a scope", async () => { + const root = await project(); + await runBaseline(cfg(), root); + await declareScope(root, ["acme::platform::**"]); + await writeFile(join(root, "metaobjects", "meta.platform.json"), PLATFORM(true), "utf8"); + + expect(await runOfflineGenerate(cfg(), root)).toBe(0); + const [dir] = await migrationDirs(root); + expect(dir).toBeDefined(); + const up = await readFile(join(root, ".metaobjects/migrations", dir!, "up.sql"), "utf8"); + expect(up).toBe(`ALTER TABLE "jobs" ADD COLUMN "note" TEXT;\n`); + }); + + test("back-compat: with NO migrate.scope the emitted SQL is unchanged", async () => { + const root = await project(); + await runBaseline(cfg(), root); + await writeFile(join(root, "metaobjects", "meta.platform.json"), PLATFORM(true), "utf8"); + + expect(await runOfflineGenerate(cfg(), root)).toBe(0); + const [dir] = await migrationDirs(root); + const up = await readFile(join(root, ".metaobjects/migrations", dir!, "up.sql"), "utf8"); + const down = await readFile(join(root, ".metaobjects/migrations", dir!, "down.sql"), "utf8"); + // Byte-for-byte what this project emitted before per-command scope existed. + expect(up).toBe(`ALTER TABLE "jobs" ADD COLUMN "note" TEXT;\n`); + expect(down).toBe(`ALTER TABLE "jobs" DROP COLUMN "note";\n`); + }); + + test("a table NO loaded object declares is still proposed for drop, scope or not", async () => { + const root = await project(); + await runBaseline(cfg(), root); + await declareScope(root, ["acme::platform::**"]); + // The arena model leaves the collection entirely — nothing declares `matches` + // any more, so migrate is back to its unchanged behaviour: propose the drop + // (blocked here, since `allow` is empty → exit 1). + await unlink(join(root, "metaobjects", "meta.arena.json")); + + expect(await runOfflineGenerate(cfg(), root)).toBe(1); + expect(await migrationDirs(root)).toHaveLength(0); + }); + + test("a scope matching only value objects and abstracts is refused before the snapshot gate", async () => { + const root = await project(); + // A package of shapes that can never declare a table: the scope matches + // loaded objects, but none the run could actually govern. + await writeFile( + join(root, "metaobjects", "meta.shared.json"), + JSON.stringify({ + "metadata.root": { + package: "acme::shared", + children: [ + { "object.entity": { name: "BaseRecord", abstract: true, children: [ + { "field.long": { name: "id" } }, + ] } }, + { "object.value": { name: "Address", children: [ + { "field.string": { name: "line1" } }, + { "field.string": { name: "line2" } }, + ] } }, + ], + }, + }), + "utf8", + ); + await declareScope(root, ["acme::shared::**"]); + // No baseline was run, so there is no snapshot: a run that got PAST the + // refusal would report "no schema snapshot" — also exit 2 — so the message + // is what pins that the scope error is the one reported, and that the + // refusal fires before the snapshot is ever read. + const errors: string[] = []; + const origErr = console.error; + console.error = (...a: unknown[]) => { errors.push(a.map(String).join(" ")); }; + try { + expect(await runOfflineGenerate(cfg(), root)).toBe(2); + } finally { + console.error = origErr; + } + const all = errors.join("\n"); + expect(all).toContain("matched none"); + expect(all).toContain("acme::shared::**"); + expect(all).toContain("acme::platform::Job"); + }); +}); diff --git a/server/typescript/packages/cli/test/unit/detect-stack.test.ts b/server/typescript/packages/cli/test/unit/detect-stack.test.ts index 4c2077531..aa26bddde 100644 --- a/server/typescript/packages/cli/test/unit/detect-stack.test.ts +++ b/server/typescript/packages/cli/test/unit/detect-stack.test.ts @@ -1,43 +1,50 @@ import { test, expect, describe } from "bun:test"; -import { mkdtempSync, mkdirSync, rmSync, writeFileSync, chmodSync } from "node:fs"; +import { mkdtempSync, mkdirSync, rmSync, writeFileSync, chmodSync, symlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { resolveStack } from "../../src/lib/detect-stack.js"; function tmp(): string { return mkdtempSync(join(tmpdir(), "detect-")); } +const REQ = JSON.stringify({ + "metadata.root": { + package: "acme", + children: [{ "requirement.functional": { name: "FR1", "@level": 1, "@status": "live" } }], + }, +}); + describe("resolveStack", () => { - test("explicit --server/--client overrides win over detection", () => { + test("explicit --server/--client overrides win over detection", async () => { const dir = tmp(); try { writeFileSync(join(dir, "package.json"), JSON.stringify({ dependencies: { "@metaobjectsdev/react": "1" } })); - const s = resolveStack(dir, { servers: ["java", "kotlin"], clients: ["tanstack"] }); + const s = await resolveStack(dir, { servers: ["java", "kotlin"], clients: ["tanstack"] }); expect(s.servers).toEqual(["java", "kotlin"]); expect(s.clients).toEqual(["tanstack"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("detects a TS server + react/tanstack from package.json deps", () => { + test("detects a TS server + react/tanstack from package.json deps", async () => { const dir = tmp(); try { writeFileSync(join(dir, "package.json"), JSON.stringify({ dependencies: { "@metaobjectsdev/cli": "1", "@metaobjectsdev/react": "1", "@metaobjectsdev/tanstack": "1" } })); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.servers).toEqual(["typescript"]); expect(s.clients.sort()).toEqual(["react", "tanstack"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("detects a Java (Maven) server from pom.xml", () => { + test("detects a Java (Maven) server from pom.xml", async () => { const dir = tmp(); try { writeFileSync(join(dir, "pom.xml"), ""); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.servers).toEqual(["java"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); describe("requirements concern (observed, not a config flag)", () => { - test("detects a requirement.* node in a nested metadata file", () => { + test("detects a requirement.* node in a nested metadata file", async () => { const dir = tmp(); try { const nested = join(dir, "metaobjects", "caps"); @@ -51,13 +58,13 @@ describe("resolveStack", () => { }, }), ); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.tokens.has("requirements")).toBe(true); expect(s.concerns).toEqual(["requirements"]); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("no requirements token for a project with no requirement.* nodes", () => { + test("no requirements token for a project with no requirement.* nodes", async () => { const dir = tmp(); try { mkdirSync(join(dir, "metaobjects"), { recursive: true }); @@ -65,21 +72,21 @@ describe("resolveStack", () => { join(dir, "metaobjects", "meta.users.json"), JSON.stringify({ "metadata.root": { children: [{ "object.entity": { name: "User" } }] } }), ); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.tokens.has("requirements")).toBe(false); expect(s.concerns).toEqual([]); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("no metaobjects/ directory at all — treated as no requirements", () => { + test("no metaobjects/ directory at all — treated as no requirements", async () => { const dir = tmp(); try { - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.tokens.has("requirements")).toBe(false); } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("survives an unreadable metadata subdirectory (defensive, no throw)", () => { + test("survives an unreadable metadata subdirectory (defensive, no throw)", async () => { const dir = tmp(); try { const locked = join(dir, "metaobjects", "locked"); @@ -87,8 +94,7 @@ describe("resolveStack", () => { writeFileSync(join(locked, "meta.caps.json"), JSON.stringify({ "requirement.functional": { name: "X" } })); chmodSync(locked, 0o000); try { - expect(() => resolveStack(dir, { servers: [], clients: [] })).not.toThrow(); - const s = resolveStack(dir, { servers: [], clients: [] }); + const s = await resolveStack(dir, { servers: [], clients: [] }); expect(s.tokens.has("requirements")).toBe(false); } finally { chmodSync(locked, 0o755); // restore so recursive cleanup below can descend into it @@ -96,16 +102,48 @@ describe("resolveStack", () => { } finally { rmSync(dir, { recursive: true, force: true }); } }); - test("concerns are observed independent of explicit --server/--client overrides", () => { + test("concerns are observed independent of explicit --server/--client overrides", async () => { const dir = tmp(); try { const nested = join(dir, "metaobjects"); mkdirSync(nested, { recursive: true }); writeFileSync(join(nested, "meta.caps.json"), JSON.stringify({ "requirement.architectural": { name: "X" } })); - const s = resolveStack(dir, { servers: ["java"], clients: [] }); + const s = await resolveStack(dir, { servers: ["java"], clients: [] }); expect(s.servers).toEqual(["java"]); expect(s.tokens.has("requirements")).toBe(true); } finally { rmSync(dir, { recursive: true, force: true }); } }); + + test("finds requirement nodes in a sources-declared tree (no metaobjects/ at the start dir)", async () => { + const dir = tmp(); + try { + mkdirSync(join(dir, ".git")); + mkdirSync(join(dir, "model"), { recursive: true }); + writeFileSync(join(dir, "model", "meta.req.json"), REQ); + mkdirSync(join(dir, "apps", "ui", ".metaobjects"), { recursive: true }); + writeFileSync( + join(dir, "apps", "ui", ".metaobjects", "config.json"), + JSON.stringify({ schema_version: 1, sources: [{ path: "../../model" }] }), + ); + const s = await resolveStack(join(dir, "apps", "ui"), { servers: [], clients: [] }); + expect(s.tokens.has("requirements")).toBe(true); + expect(s.concerns).toContain("requirements"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); + + test("finds requirement nodes behind a NESTED symlinked directory", async () => { + const dir = tmp(); + try { + mkdirSync(join(dir, ".git")); + mkdirSync(join(dir, "real"), { recursive: true }); + writeFileSync(join(dir, "real", "meta.req.json"), REQ); + mkdirSync(join(dir, "metaobjects"), { recursive: true }); + writeFileSync(join(dir, "metaobjects", "meta.a.json"), "{}"); + symlinkSync(join(dir, "real"), join(dir, "metaobjects", "linked"), "dir"); + const s = await resolveStack(dir, { servers: [], clients: [] }); + expect(s.tokens.has("requirements")).toBe(true); + expect(s.concerns).toContain("requirements"); + } finally { rmSync(dir, { recursive: true, force: true }); } + }); }); }); diff --git a/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts b/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts index dcba4d83a..91de41102 100644 --- a/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts +++ b/server/typescript/packages/codegen-ts/src/projection/build-projection-views.ts @@ -53,6 +53,14 @@ export interface ExpectedView { name: string; schema?: string; sql: string; + /** + * `resolutionKey()` of the object that declared this view — the projection, or the + * write-through entity hosting its own read view. migrate-ts records it as + * PROVENANCE (never onto the view descriptor, never into the committed snapshot) so + * a per-command `migrate.scope` can decide ownership on the declaring FQN rather + * than on the physical view name, which no naming strategy can reverse. + */ + fqn: string; /** * Physical tables this view reads (base + every joined table). The migrate-ts * diff uses this to recreate the view when one of its source tables undergoes a @@ -199,6 +207,7 @@ function emitViewFor( sql: body, dependsOn, columns, + fqn: host.resolutionKey(), ...(schema !== undefined ? { schema } : {}), }); } @@ -240,6 +249,7 @@ function emitSqlView( name: source.physicalName, // FR-016 four-step physical name sql: source.sqlBody!, // verbatim — never parsed, never re-wrapped dependsOn, + fqn: host.resolutionKey(), // columns OMITTED → "unknown" → gated drop+create fail-safe. ...(schema !== undefined ? { schema } : {}), }); diff --git a/server/typescript/packages/codegen-ts/src/runner.ts b/server/typescript/packages/codegen-ts/src/runner.ts index 73e645f8c..7a18f0f37 100644 --- a/server/typescript/packages/codegen-ts/src/runner.ts +++ b/server/typescript/packages/codegen-ts/src/runner.ts @@ -60,6 +60,41 @@ export interface RunGenOpts { * `--dry-run`, and watching it reappear. */ dryRun?: boolean; + /** + * Output scope — an object is generated only when this predicate returns true + * for its fully-qualified name (`obj.resolutionKey()`, `::`). + * Intersects with `entityFilter`: both must pass. Absent ⇒ every object is + * in scope (byte-identical to a project with no `scope` declared). + * + * The collection metadata always loads in FULL regardless of this predicate — + * scope filters OUTPUT, never input (design §4.3). So an in-scope object may + * reference an out-of-scope one (an FK target, a relationship `@objectRef`, a + * projection's base) and resolve perfectly at load time, while the code + * emitted FOR the in-scope object still imports/names a symbol that was never + * generated. This is left silent by design, not auto-widened: the adopter + * declared the scope precisely because something else (another consumer, + * another codegen run) owns those objects, and the reference is real. Warning + * on it correctly would require walking every reference kind (identity.reference, + * every relationship.* @objectRef, projection extends bases, field.object + * @objectRef) FQN-resolved against the SAME scope — genuinely new machinery, + * not a fit for the existing `warnings: string[]` channel at this seam. If an + * adopter hits it, the failure is a plain compiler error in the generated + * code (an unresolved import) — loud, at build time, not silent at runtime. + * + * Deliberately a PLAIN PREDICATE, not the `include`/`exclude` pattern strings + * `@metaobjectsdev/sdk`'s `scope.ts` compiles. `codegen-ts` must not depend on + * `@metaobjectsdev/sdk` — the dependency runs the other way (`cli` depends on + * both) — so it cannot import `matchesScope`/`CompiledScope` itself. The + * design's "package patterns, never a predicate function" rule (§4.3 of the + * metadata-source-resolution design doc) governs CONFIG SURFACES that must + * port identically to a `pom.xml` / `metaobjects.config.yaml` in every + * language port; it says nothing about internal plumbing between two + * TypeScript packages in this one repo. Do not "fix" this into a config + * shape — `cli`'s `gen`/`verify` commands are the only callers, and a + * `Collection` already exposes exactly this predicate as `inScope`, which + * they pass straight through. + */ + scope?: (fqn: string) => boolean; } export interface RunGenResult { @@ -136,16 +171,46 @@ export async function runGen(opts: RunGenOpts): Promise { } const root = opts.metadata; - // 1. Resolve entities (filter + safety check). + // 1. Resolve entities (entityFilter + scope + safety check). This is the + // single choke point for entity selection — scope INTERSECTS entityFilter + // (an object must pass both), matched against the object's + // fully-qualified name (resolutionKey(), never the bare name — two + // packages may declare the same short name). const allObjects = root.objects(); const entityFilter = opts.entityFilter; - const filtered = entityFilter + const afterEntityFilter = entityFilter ? allObjects.filter((o) => entityFilter.includes(o.name)) : allObjects; + const scope = opts.scope; + const filtered = scope + ? afterEntityFilter.filter((o) => scope(o.resolutionKey())) + : afterEntityFilter; if (filtered.length === 0) { - const reason = opts.entityFilter - ? "no object children match the provided entityFilter" - : "root has no object children"; + // Name the REAL cause. When `scope` is absent, this is byte-identical to + // the pre-scope two-way branch (kept as its own arm, rather than folded + // into the scope-aware logic below, so an unscoped project's warning text + // — including its quirky edge case: an empty root with entityFilter set + // still blames entityFilter — is untouched). Only when `scope` is + // present does a THIRD reason become reachable: "root has no object + // children" for a scoped-out model, or "...entityFilter" for a scope + // that admitted everything entityFilter then excluded, are both false + // statements that send the reader to the wrong file. + let reason: string; + if (scope === undefined) { + // Byte-identical to the pre-scope branch, quirk included: an EMPTY root + // with an entityFilter set still blames the filter. Wrong, and untouched + // — changing what an unscoped project reads is a behaviour change, and + // this is a shape change. + reason = entityFilter + ? "no object children match the provided entityFilter" + : "root has no object children"; + } else if (allObjects.length === 0) { + reason = "root has no object children"; + } else if (afterEntityFilter.length === 0) { + reason = "no object children match the provided entityFilter"; + } else { + reason = "no object children match the configured scope"; + } warnings.push(`No entities to generate — ${reason}.`); return { files: [], warnings, conflicts: [] }; } diff --git a/server/typescript/packages/codegen-ts/test/run-gen.test.ts b/server/typescript/packages/codegen-ts/test/run-gen.test.ts index 3116d7b1a..d3731257c 100644 --- a/server/typescript/packages/codegen-ts/test/run-gen.test.ts +++ b/server/typescript/packages/codegen-ts/test/run-gen.test.ts @@ -333,3 +333,145 @@ describe("runGen — entityFilter", () => { expect(existsSync(join(tmp, "index.ts"))).toBe(false); }); }); + +// --------------------------------------------------------------------------- +// Task 12b: collection-level `scope` filters generated output. +// --------------------------------------------------------------------------- +describe("runGen — scope", () => { + test("a scope predicate emits artifacts for in-scope entities only (file list + contents)", async () => { + const loader = new MetaDataLoader(); + const result = await loader.load([new FileSource(join(FIXTURE_DIR, "two-entities-fk.json"))]); + expect(result.errors).toEqual([]); + + const out = await runGen({ + config: defineConfig({ + outDir: tmp, + extStyle: "none", + dbImport: "~/server/db", + dialect: "postgres", + generators: [entityFile(), queriesFile(), barrel()], + }), + metadata: result.root, + // "demo::Post" only — User is in the same package but declares an + // `identity.reference`/`relationship.association` TO Post's declared + // scope, so this fixture doubles as evidence the predicate is matched + // against the FQN (`demo::Post`), never the bare "Post". + scope: (fqn) => fqn === "demo::Post", + }); + + expect(out.warnings).toEqual([]); + expect(existsSync(join(tmp, "Post.ts"))).toBe(true); + expect(existsSync(join(tmp, "Post.queries.ts"))).toBe(true); + expect(existsSync(join(tmp, "User.ts"))).toBe(false); + expect(existsSync(join(tmp, "User.queries.ts"))).toBe(false); + + const postContent = readFileSync(join(tmp, "Post.ts"), "utf-8"); + expect(postContent).toContain("pgTable"); + expect(postContent).toContain("authorId"); + + // Barrel content, not just the file's existence — only Post is re-exported. + const barrelContent = readFileSync(join(tmp, "index.ts"), "utf-8"); + expect(barrelContent).toContain('export * from "./Post"'); + expect(barrelContent).not.toContain("User"); + }); + + test("no scope option produces byte-identical output to an always-matching scope predicate", async () => { + const loader = new MetaDataLoader(); + const result = await loader.load([new FileSource(join(FIXTURE_DIR, "two-entities-fk.json"))]); + expect(result.errors).toEqual([]); + + const noScopeDir = join(tmp, "no-scope"); + const alwaysTrueDir = join(tmp, "always-true"); + + const baseConfig = { + extStyle: "none" as const, + dbImport: "~/server/db", + dialect: "postgres" as const, + generators: [entityFile(), queriesFile(), routesFile(), barrel()], + }; + + // The real-world "no scope declared" path (`meta gen`) still ALWAYS passes a + // predicate — `collection.inScope`, which an unconfigured project compiles + // from an empty include/exclude, so it admits everything. The byte-identical + // guarantee that matters is exactly this: omitting `scope` entirely vs. a + // predicate that matches every entity must produce identical output, not + // merely "close". + const outA = await runGen({ + config: defineConfig({ ...baseConfig, outDir: noScopeDir }), + metadata: result.root, + }); + const outB = await runGen({ + config: defineConfig({ ...baseConfig, outDir: alwaysTrueDir }), + metadata: result.root, + scope: () => true, + }); + + expect(outB.warnings).toEqual(outA.warnings); + + const filesA = readdirSync(noScopeDir).sort(); + const filesB = readdirSync(alwaysTrueDir).sort(); + expect(filesB).toEqual(filesA); + for (const f of filesA) { + expect(readFileSync(join(alwaysTrueDir, f), "utf-8")).toEqual( + readFileSync(join(noScopeDir, f), "utf-8"), + ); + } + }); + + test("scope intersects entityFilter — an entity passing only one of the two is not emitted", async () => { + const loader = new MetaDataLoader(); + const result = await loader.load([new FileSource(join(FIXTURE_DIR, "two-entities-fk.json"))]); + expect(result.errors).toEqual([]); + + // User passes entityFilter but is excluded by scope (which admits only + // Post); Post passes scope but is excluded by entityFilter. Neither alone + // is enough — intersection means NEITHER is emitted. + const out = await runGen({ + config: defineConfig({ + outDir: tmp, + extStyle: "none", + dbImport: "~/server/db", + dialect: "postgres", + generators: [entityFile(), queriesFile(), barrel()], + }), + metadata: result.root, + entityFilter: ["User"], + scope: (fqn) => fqn === "demo::Post", + }); + + expect(out.files).toHaveLength(0); + expect(existsSync(join(tmp, "User.ts"))).toBe(false); + expect(existsSync(join(tmp, "Post.ts"))).toBe(false); + // Attributed to the real cause (scope), not entityFilter — User genuinely + // matched entityFilter and was then excluded by scope. + expect(out.warnings.some((w) => w.includes("scope"))).toBe(true); + }); + + test("a scope matching nothing warns with a reason that names scope", async () => { + const loader = new MetaDataLoader(); + const result = await loader.load([new FileSource(join(FIXTURE_DIR, "two-entities-fk.json"))]); + expect(result.errors).toEqual([]); + + const out = await runGen({ + config: defineConfig({ + outDir: tmp, + extStyle: "none", + dbImport: "~/server/db", + dialect: "postgres", + generators: [entityFile(), queriesFile(), barrel()], + }), + metadata: result.root, + scope: () => false, + }); + + expect(out.files).toHaveLength(0); + // The reason must name scope specifically — "root has no object children" + // would be a false statement (the root has two) that sends the reader to + // the wrong file. + const scopeWarning = out.warnings.find((w) => w.includes("No entities to generate")); + expect(scopeWarning).toBeDefined(); + expect(scopeWarning).toContain("scope"); + expect(scopeWarning).not.toContain("entityFilter"); + expect(scopeWarning).not.toContain("root has no object children"); + }); +}); diff --git a/server/typescript/packages/docs-site/src/load.ts b/server/typescript/packages/docs-site/src/load.ts index a3d40d81e..5ee9b171d 100644 --- a/server/typescript/packages/docs-site/src/load.ts +++ b/server/typescript/packages/docs-site/src/load.ts @@ -1,6 +1,6 @@ import { mkdtempSync, readdirSync, rmSync, statSync, symlinkSync } from "node:fs"; import { tmpdir } from "node:os"; -import { basename, join, resolve } from "node:path"; +import { basename, dirname, join, resolve } from "node:path"; import { MetaDataLoader, composeRegistry, coreTypesProvider, dbProvider, docProvider, promptProvider, uiProvider } from "@metaobjectsdev/metadata"; import type { MetaData, MetaRoot, MetaDataTypeProvider } from "@metaobjectsdev/metadata"; import { FileSource } from "@metaobjectsdev/metadata/core"; @@ -26,14 +26,19 @@ export async function loadModel( ): Promise { const staging = mkdtempSync(join(tmpdir(), "metadocs-")); try { - const usedBasenames = new Set(); + // Source groups are keyed by the staging entry's name, so two source dirs + // sharing a basename need distinct names — `metaobjects/` plus a shared + // model's `../shared-model/metaobjects` is the ordinary shape of a + // multi-source project, and refusing it would make the feature unusable + // with `--site`. Collisions are qualified by their parent directory (the + // name a reader recognises) before falling back to a counter. + const used = new Set(); + const groupNames: string[] = []; for (const dir of sourceDirs) { - const baseName = basename(dir); - if (usedBasenames.has(baseName)) { - throw new Error(`duplicate source dir basename: ${baseName}`); - } - usedBasenames.add(baseName); - symlinkSync(resolve(dir), join(staging, baseName)); + const name = uniqueGroupName(dir, used); + used.add(name); + groupNames.push(name); + symlinkSync(resolve(dir), join(staging, name)); } const registry = composeRegistry([coreTypesProvider, dbProvider, docProvider, promptProvider, uiProvider, ...extraProviders]); // Feed files in files-before-subdirs order (the same order the sdk's loadMemory @@ -53,13 +58,35 @@ export async function loadModel( return { root: result.root, warnings: result.warnings.map((w) => w.message), - sourceDirs: sourceDirs.map((d) => basename(resolve(d))), + // The staging entry names, NOT raw basenames: `treeOf` matches a node's + // source path segment against this list, and a collision-qualified name + // is what that segment actually is. + sourceDirs: groupNames, }; } finally { rmSync(staging, { recursive: true, force: true }); } } +/** + * A staging-directory entry name for `dir` that no earlier source dir already + * took. The basename when it is free (so a single-source project's group name + * is unchanged); otherwise `-`, then a counter — deterministic + * for a given source list, which keeps the emitted site byte-stable. + */ +function uniqueGroupName(dir: string, used: ReadonlySet): string { + const abs = resolve(dir); + const base = basename(abs); + if (!used.has(base)) return base; + const parent = basename(dirname(abs)); + const qualified = parent === "" || parent === base ? base : `${parent}-${base}`; + if (!used.has(qualified)) return qualified; + for (let n = 2; ; n++) { + const candidate = `${qualified}-${n}`; + if (!used.has(candidate)) return candidate; + } +} + /** Metadata files under `dir`, files-before-subdirs with each level sorted — the * overlay-safe order the sdk's loadMemory uses, so a base loads before an overlay * nested under it. Symlinks (the staging dir uses them) are followed. */ diff --git a/server/typescript/packages/metadata/src/errors.ts b/server/typescript/packages/metadata/src/errors.ts index 70beb35a7..7fa3c6eed 100644 --- a/server/typescript/packages/metadata/src/errors.ts +++ b/server/typescript/packages/metadata/src/errors.ts @@ -208,6 +208,18 @@ export const ERROR_CODES = [ // an integer array. An array-of-enum stays string-backed: drop @intValueMap, // or make the field scalar. "ERR_ENUM_INT_VALUE_MAP_ARRAY", + // Phase-1 metadata-source-resolution — a path source declared in + // .metaobjects/config.json does not exist on disk. + "ERR_SOURCE_UNRESOLVED", + // Phase-1 metadata-source-resolution — a declared source kind (resource or + // package) is not supported by this toolchain. + "ERR_SOURCE_KIND_UNSUPPORTED", + // Phase-1 metadata-source-resolution — a scope include/exclude package + // pattern is malformed (empty pattern or empty :: segment). + "ERR_SCOPE_PATTERN_INVALID", + // Phase-1 metadata-source-resolution — no metadata collection was discovered: + // no config declaring sources, and no default metaobjects/ directory. + "ERR_COLLECTION_NOT_FOUND", "ERR_UNKNOWN", ] as const; diff --git a/server/typescript/packages/metadata/test/errors.test.ts b/server/typescript/packages/metadata/test/errors.test.ts index 2eba627e1..9b1d087f6 100644 --- a/server/typescript/packages/metadata/test/errors.test.ts +++ b/server/typescript/packages/metadata/test/errors.test.ts @@ -28,3 +28,16 @@ test("MetaModelError carries a stable ERR_PROVIDER_* code", () => { expect(thrown).toBeInstanceOf(MetaModelError); expect((thrown as MetaModelError).code).toBe("ERR_PROVIDER_DUPLICATE_ID"); }); + +// Phase-1 metadata-source-resolution design: register error codes that will be +// raised when loading sources from .metaobjects/config.json. +test("phase-1 source-resolution error codes are registered in the shared ledger", () => { + for (const code of [ + "ERR_SOURCE_UNRESOLVED", + "ERR_SOURCE_KIND_UNSUPPORTED", + "ERR_SCOPE_PATTERN_INVALID", + "ERR_COLLECTION_NOT_FOUND", + ] as const) { + expect(ERROR_CODES).toContain(code); + } +}); diff --git a/server/typescript/packages/migrate-ts/src/diff/index.ts b/server/typescript/packages/migrate-ts/src/diff/index.ts index 4da7ea67f..f5b3d7a1b 100644 --- a/server/typescript/packages/migrate-ts/src/diff/index.ts +++ b/server/typescript/packages/migrate-ts/src/diff/index.ts @@ -15,6 +15,7 @@ import { viewReplaceIsLegal } from "../view-column-types.js"; import { checkExprEquals, normalizeCheckExpr } from "../check-expr-compare.js"; import { isPgAutoSequenceDefault } from "../pg-identity-default.js"; import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; +import { qualifiedDbName } from "../qualified-name.js"; export interface DiffArgs { expected: SchemaSnapshot; @@ -90,10 +91,12 @@ const DEFAULT_IGNORE_TABLES: string[] = [ * * For SQLite (no schema concept), every table has schema=undefined, so this maps * all tables to the same "public." prefix — harmless and preserves existing behavior. + * + * `qualifiedDbName` is THE definition (qualified-name.ts): the act-side exclusion + * sets — declared-`@unmanaged` and out-of-scope — are matched against these keys, so + * a second spelling here would silently un-suppress an object and propose its drop. */ -function tableIdentity(table: { name: string; schema?: string }): string { - return (table.schema ?? DEFAULT_DB_SCHEMA_POSTGRES) + "." + table.name; -} +const tableIdentity = qualifiedDbName; /** * Build the optional-schema spread used when constructing Change records. @@ -590,9 +593,7 @@ function diffTableChecks( } } -function viewIdentity(v: { name: string; schema?: string }): string { - return (v.schema ?? DEFAULT_DB_SCHEMA_POSTGRES) + "." + v.name; -} +const viewIdentity = qualifiedDbName; /** * Decide, per view, whether the DB matches the model. diff --git a/server/typescript/packages/migrate-ts/src/drift/drift.ts b/server/typescript/packages/migrate-ts/src/drift/drift.ts index f5369ee83..eb068c02a 100644 --- a/server/typescript/packages/migrate-ts/src/drift/drift.ts +++ b/server/typescript/packages/migrate-ts/src/drift/drift.ts @@ -16,10 +16,11 @@ import type { Kysely } from "kysely"; import type { MetaRoot } from "@metaobjectsdev/metadata"; import type { ColumnNamingStrategy } from "@metaobjectsdev/metadata"; -import { buildExpectedSchema } from "../expected-schema.js"; +import { buildExpectedSchemaWithProvenance } from "../expected-schema.js"; import { introspect } from "../introspect/index.js"; import { diff } from "../diff/index.js"; import { collectUnmanagedNames } from "../unmanaged.js"; +import { scopeExpectedSchema, scopedDiffInputs, type ObjectScopePredicate } from "../scope.js"; import type { AllowOptions, Dialect, DiffResult, SchemaSnapshot } from "../types.js"; export interface ComputeDriftOptions { @@ -46,6 +47,36 @@ export interface ComputeDriftOptions { * itself; pass these so view drift is detected. Defaults to none. */ views?: readonly import("../expected-schema.js").ExpectedViewInput[]; + /** + * Per-command scope (`migrate.scope`): objects whose declaring FQN this predicate + * rejects are governed by somebody else. They leave the expected side AND are + * suppressed on the actual side, so their divergence is neither drift nor a + * proposed drop — `verify` reports them as out-of-scope instead (see + * `DriftResult.outOfScope`). Omit to govern everything loaded (unchanged behavior). + * + * `verify --db` and `migrate` deliberately share ONE declaration: a drift gate + * failing on tables migrate does not own is incoherent. + */ + inScope?: ObjectScopePredicate; +} + +export interface DriftResult extends DiffResult { + /** + * Qualified physical names excluded by `inScope` — empty when no scope was + * given. The caller REPORTS these: an object silently dropped from the + * comparison is indistinguishable from one that was checked and found clean. + */ + outOfScope: readonly string[]; + /** + * The schemas this comparison governed (`ScopedExpectedSchema.declaredSchemas`), + * `undefined` when no scope was given and `diff` derived its own. + * + * Reported so a SECOND comparison over the same run — `verify`'s committed-snapshot + * gate — can govern exactly the same schemas instead of re-deriving them from a + * different expected side. Together with `outOfScope` this pair is a + * `GovernedScope`, which is what `excludeFromSnapshot` takes. + */ + declaredSchemas: readonly string[] | undefined; } /** @@ -65,24 +96,33 @@ export async function computeDriftFromActual( dialect: Dialect, metadata: MetaRoot, opts?: ComputeDriftOptions, -): Promise { - const expected = buildExpectedSchema(metadata, { - dialect, - ...(opts?.columnNamingStrategy !== undefined - ? { columnNamingStrategy: opts.columnNamingStrategy } - : {}), - ...(opts?.views !== undefined ? { views: opts.views } : {}), - }); - return diff({ - expected, +): Promise { + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(metadata, { + dialect, + ...(opts?.columnNamingStrategy !== undefined + ? { columnNamingStrategy: opts.columnNamingStrategy } + : {}), + ...(opts?.views !== undefined ? { views: opts.views } : {}), + }), + opts?.inScope, + ); + const result = await diff({ + // The three scoped-diff obligations as one value (see scope.ts's header): + // the narrowed expected side, `unmanagedNames` merging @unmanaged with the + // out-of-scope names so neither is proposed for drop, and the schema scope + // pinned to the UNSCOPED model so a narrow scope can never widen the run. + ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, dialect, allow: opts?.allow ?? {}, - // #208 §7 — a declared-@unmanaged object is external, so it is not drift: exclude it - // from the actual side (same as `meta migrate`) rather than surface a false drop-*. - unmanagedNames: collectUnmanagedNames(metadata), ...(opts?.ignoreTables !== undefined ? { ignoreTables: opts.ignoreTables } : {}), }); + return { + ...result, + outOfScope: scoped.outOfScope, + declaredSchemas: scoped.declaredSchemas, + }; } /** @@ -97,7 +137,7 @@ export async function computeDrift( dialect: Dialect, metadata: MetaRoot, opts?: ComputeDriftOptions, -): Promise { +): Promise { const actual = await introspect(db, dialect); return computeDriftFromActual(actual, dialect, metadata, opts); } diff --git a/server/typescript/packages/migrate-ts/src/expected-schema.ts b/server/typescript/packages/migrate-ts/src/expected-schema.ts index fe25f1cbb..8c32e675a 100644 --- a/server/typescript/packages/migrate-ts/src/expected-schema.ts +++ b/server/typescript/packages/migrate-ts/src/expected-schema.ts @@ -61,6 +61,7 @@ import type { Dialect, SchemaSnapshot, TableDescriptor, ColumnDescriptor, IndexDescriptor, FkDescriptor, CheckDescriptor, ViewDescriptor, } from "./types.js"; +import { qualifiedDbName } from "./qualified-name.js"; import { viewFingerprint } from "./view-fingerprint.js"; import { resolveViewColumns, type ExpectedViewColumnInput } from "./view-column-types.js"; import { @@ -112,12 +113,55 @@ export interface ExpectedViewInput { sql?: string; dependsOn?: readonly string[]; columns?: readonly ExpectedViewColumnInput[]; + /** + * `resolutionKey()` of the object that declared this view — its PROVENANCE. + * Recorded in the provenance map and deliberately NEVER copied onto the + * `ViewDescriptor`: descriptors are serialized into the committed snapshot, and + * a descriptor that gains a field owes a `SNAPSHOT_FORMAT_VERSION` bump, which + * hard-fails every older reader. Optional — a caller that supplies no FQN gets a + * view with no provenance, which `scopeExpectedSchema` keeps (never guesses). + */ + fqn?: string; } +/** + * Qualified physical name (`qualifiedDbName`) → the `resolutionKey()` of the + * metadata object that declared it. The ONLY sound basis for a per-command scope + * decision: a SQL name cannot be reversed into an FQN (naming strategies, `@table` + * overrides and TPH folding are all lossy), and a second metadata walk would have + * to re-implement Pass 1's skip rules — abstract, TPH subtype, no writable source, + * `@unmanaged` — and would drift from them. + */ +export type SchemaProvenance = ReadonlyMap; + +export interface ExpectedSchemaWithProvenance { + snapshot: SchemaSnapshot; + provenance: SchemaProvenance; +} + +/** + * The expected schema as every existing caller wants it. Thin wrapper over + * {@link buildExpectedSchemaWithProvenance}; byte-identical output. + */ export function buildExpectedSchema( root: MetaData, opts?: BuildExpectedSchemaOptions, ): SchemaSnapshot { + return buildExpectedSchemaWithProvenance(root, opts).snapshot; +} + +/** + * The expected schema PLUS the declaring FQN of every table and view in it. + * + * Provenance is threaded out of the passes that already hold the declaring node — + * Pass 2 has each table's entity, Pass 4 each view's input — so there is exactly + * one walk and one set of skip rules. Callers that filter by scope + * (`scopeExpectedSchema`) consume it; callers that don't use the wrapper above. + */ +export function buildExpectedSchemaWithProvenance( + root: MetaData, + opts?: BuildExpectedSchemaOptions, +): ExpectedSchemaWithProvenance { // D1 is SQLite at the SQL level; normalize it so downstream dialect checks // don't need to handle "d1" separately. const dialect = opts?.dialect === "d1" ? "sqlite" : opts?.dialect; @@ -207,6 +251,12 @@ export function buildExpectedSchema( return byBareHit === AMBIGUOUS ? undefined : byBareHit; }; + // Provenance: qualified physical name → declaring object's FQN. Recorded as the + // descriptors are built, never re-derived from a SQL name (lossy) and never by a + // second walk (it would have to duplicate Pass 1's skip rules and would drift from + // them — a TPH subtype, for one, shares its base's table and declares none of its own). + const provenance = new Map(); + // Pass 2: build full descriptors with FK resolution. // Schema is resolved here (not stored in Pass 1) to avoid exactOptionalPropertyTypes // issues with `string | undefined` vs `schema?: string`. @@ -214,6 +264,7 @@ export function buildExpectedSchema( const t = buildTable(entity, tableName, resolveTargetTable, root as MetaRoot, strategy, dialect); const schema = resolveTableSchema(entity); if (schema !== undefined) t.schema = schema; + provenance.set(qualifiedDbName(t), entity.resolutionKey()); return t; }); @@ -288,13 +339,17 @@ export function buildExpectedSchema( // whether a view change can use a non-destructive CREATE OR REPLACE. const views: ViewDescriptor[] = (opts?.views ?? []).map((v) => { const columns = resolveViewColumns(v.columns, tables); - return { + const descriptor: ViewDescriptor = { name: v.name, ...(v.schema !== undefined ? { schema: v.schema } : {}), ...(v.sql !== undefined ? { sql: v.sql, fingerprint: viewFingerprint(v.sql) } : {}), ...(v.dependsOn !== undefined ? { dependsOn: v.dependsOn } : {}), ...(columns !== undefined ? { columns } : {}), }; + // The declaring FQN goes to the provenance map ONLY — never onto the descriptor, + // which is what the committed snapshot serializes (see ExpectedViewInput.fqn). + if (v.fqn !== undefined) provenance.set(qualifiedDbName(descriptor), v.fqn); + return descriptor; }); // Collision guard: two DISTINCT metadata objects that resolve to the same generated @@ -324,7 +379,7 @@ export function buildExpectedSchema( ); } - return { tables, views }; + return { snapshot: { tables, views }, provenance }; } /** diff --git a/server/typescript/packages/migrate-ts/src/index.ts b/server/typescript/packages/migrate-ts/src/index.ts index d09c47d6f..b4c8c9208 100644 --- a/server/typescript/packages/migrate-ts/src/index.ts +++ b/server/typescript/packages/migrate-ts/src/index.ts @@ -8,11 +8,23 @@ // See docs/specs/2026-05-11-v0.2-sp4-migrate-ts-design.md. // Pipeline functions -export { buildExpectedSchema } from "./expected-schema.js"; +export { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "./expected-schema.js"; +export type { ExpectedSchemaWithProvenance, SchemaProvenance } from "./expected-schema.js"; export { introspect, introspectPostgres, introspectSqlite } from "./introspect/index.js"; export { diff } from "./diff/index.js"; export { collectUnmanagedNames } from "./unmanaged.js"; -export { computeDrift, computeDriftFromActual, type ComputeDriftOptions } from "./drift/drift.js"; +// Per-command scope (`migrate.scope`) — see scope.ts for why the suppression is +// two-sided and why the pattern engine stays in @metaobjectsdev/sdk. +export { + scopeExpectedSchema, + declaredSchemasOf, + carryForwardOutOfScope, + excludeFromSnapshot, + scopedDiffInputs, +} from "./scope.js"; +export type { ObjectScopePredicate, ScopedExpectedSchema, GovernedScope } from "./scope.js"; +export { qualifiedDbName } from "./qualified-name.js"; +export { computeDrift, computeDriftFromActual, type ComputeDriftOptions, type DriftResult } from "./drift/drift.js"; export { classifyDrift, driftAgainstSnapshot } from "./drift/classify.js"; export type { DriftClassification } from "./drift/classify.js"; export { emit } from "./emit/index.js"; diff --git a/server/typescript/packages/migrate-ts/src/qualified-name.ts b/server/typescript/packages/migrate-ts/src/qualified-name.ts new file mode 100644 index 000000000..b702b5742 --- /dev/null +++ b/server/typescript/packages/migrate-ts/src/qualified-name.ts @@ -0,0 +1,22 @@ +// The ONE qualified-physical-name form: `.`, with an absent schema +// normalized to the Postgres default. +// +// Three things must key DB objects identically or the diff silently disagrees with +// itself: `diff`'s table/view identity maps, the declared-`@unmanaged` exclusion set +// (`collectUnmanagedNames`), and the out-of-scope exclusion set (`scopeExpectedSchema`). +// The last two are ACT-side suppressions matched against the first, so a name built a +// second way — a different default schema, a different separator — reads as "not +// suppressed" and the object it names comes back as a proposed DROP. One function. +// +// SQLite has no schema concept, so every SQLite object normalizes to the same prefix. +// That is harmless: it is a constant, and the un-prefixed names were already unique. + +import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; + +/** `.`; an absent schema is the Postgres default (`public`). The + * parameter accepts an EXPLICIT `undefined` schema (not only an omitted key) so a + * caller holding a `string | undefined` can pass it straight through under + * `exactOptionalPropertyTypes` — the two spell the same thing here. */ +export function qualifiedDbName(obj: { name: string; schema?: string | undefined }): string { + return `${obj.schema ?? DEFAULT_DB_SCHEMA_POSTGRES}.${obj.name}`; +} diff --git a/server/typescript/packages/migrate-ts/src/scope.ts b/server/typescript/packages/migrate-ts/src/scope.ts new file mode 100644 index 000000000..0fa36ed27 --- /dev/null +++ b/server/typescript/packages/migrate-ts/src/scope.ts @@ -0,0 +1,261 @@ +// Per-command scope — narrowing a migrate/verify run to the objects it governs. +// +// A consumer sharing a database with another owner declares +// `"migrate": { "scope": ["acme::platform::**"] }`. Tables and views outside that +// scope are neither created nor dropped, which takes TWO suppressions: +// +// 1. drop them from the EXPECTED side, so nothing is created or altered; +// 2. suppress the same names on the ACTUAL side (via `diff`'s `unmanagedNames`, +// the seam `@unmanaged` already uses), so nothing is dropped. +// +// Doing only (1) is strictly worse than doing nothing: every out-of-scope table that +// EXISTS in the database becomes a proposed `DROP TABLE` — the precise hazard this +// feature exists to remove. `scopeExpectedSchema` therefore returns both halves and +// callers must thread `outOfScope` into the diff. +// +// There is a THIRD half, and it is the one that bites hardest when the scope is +// wrong. `diff` derives its SCHEMA scope from the schemas the expected side +// mentions, falling back to "no schema scoping at all" when expected is empty (the +// legacy whole-DB path for a project with no model). A scope matching NOTHING +// empties `expected`, reaches that fallback, and every actual table in every schema +// becomes a drop candidate — another owner's included, which was never in `expected` +// so it has no provenance and never lands in `outOfScope`. Narrowing must never +// WIDEN. `declaredSchemas` below reports the UNSCOPED model's schemas so callers can +// pin `diff`'s `scopeSchemas` to a property of the whole model, which `migrate.scope` +// then cannot move in either direction. +// +// THE RULE THAT FOLLOWS FROM THAT, stated once because it is easy to read the other +// way: **a scope narrows which OBJECTS the tool governs, never which SCHEMAS it is +// allowed to see.** Pinning `scopeSchemas` to the unscoped model means a scope that +// excludes every declared object in schema `X` leaves `X` in scope, so another +// owner's UNDECLARED table in `X` stays a drop candidate — exactly as it would be on +// an unscoped run of the same model. That is deliberate: a schema this model +// declares into is a schema this model manages, and deriving the schema set from the +// survivors instead is precisely the inversion above. Declaring a scope is not a way +// to hand a schema over; removing the objects from the model is. +// +// `scopedDiffInputs` exists so no caller has to remember any of this: it returns all +// three obligations as one object, and every scoped `diff` call goes through it. + +import { DEFAULT_DB_SCHEMA_POSTGRES } from "@metaobjectsdev/metadata"; +import type { DiffArgs } from "./diff/index.js"; +import type { ExpectedSchemaWithProvenance } from "./expected-schema.js"; +import { qualifiedDbName } from "./qualified-name.js"; +import type { SchemaSnapshot } from "./types.js"; + +/** + * Decides whether an object's fully-qualified name (`resolutionKey()`) is governed + * by this run. Supplied by the caller as a PREDICATE so migrate-ts never carries a + * second implementation of the scope-pattern grammar — `matchesScope` in + * `@metaobjectsdev/sdk` is the only one, and the CLI adapts a compiled scope to + * this seam. + */ +export type ObjectScopePredicate = (fqn: string) => boolean; + +export interface ScopedExpectedSchema { + /** The expected schema narrowed to the governed objects. */ + snapshot: SchemaSnapshot; + /** + * Qualified physical names (`.`) of the tables and views removed + * above. Reaches `diff`'s `unmanagedNames` (MERGED with `collectUnmanagedNames`, + * never replacing it) so the actual side is suppressed too — `scopedDiffInputs` + * does that merge; see the module header for why omitting it inverts the feature. + */ + outOfScope: string[]; + /** + * The database schemas the UNSCOPED model declares, for `diff`'s `scopeSchemas`. + * `scopedDiffInputs` threads it — see the module header: without it a scope + * matching nothing hands `diff` an empty expected side, which it reads as "no + * model, govern the whole database". + * + * `undefined` when no predicate was supplied (so `diff` derives its own set from + * an untouched `expected`, exactly as before — an unscoped project's arguments are + * unchanged) and also when the unscoped model declares no tables or views at all + * (nothing to derive from; `diff`'s legacy whole-DB fallback is preserved). + */ + declaredSchemas?: string[]; +} + +/** + * Carry an out-of-scope object forward into the snapshot a run is about to commit. + * + * The committed snapshot is built from the metadata-expected schema, which a scoped + * run has already narrowed — so accepting a scoped run DELETES every out-of-scope + * entry the previous snapshot held. Widening or removing `migrate.scope` later then + * proposes `CREATE TABLE` for a table that exists, and the migration fails at apply. + * + * `prior` is the snapshot (or introspected schema) the run diffed against, and the + * entries taken from it are exactly the ones this run excluded — nothing else is + * carried, so a table the model never declared is unaffected either way. An empty + * `outOfScope` returns the SAME object, so an unscoped run commits a byte-identical + * snapshot. + */ +export function carryForwardOutOfScope( + next: SchemaSnapshot, + prior: SchemaSnapshot, + outOfScope: readonly string[], +): SchemaSnapshot { + if (outOfScope.length === 0) return next; + const excluded = new Set(outOfScope); + return { + ...next, + tables: [...next.tables, ...splitOnName(prior.tables, excluded).named], + views: [...next.views, ...splitOnName(prior.views, excluded).named], + }; +} + +/** + * Drop the out-of-scope entries from a COMMITTED SNAPSHOT, producing the same + * three-part shape `scopeExpectedSchema` produces so the result can go straight + * through {@link scopedDiffInputs}. + * + * `verify`'s committed-snapshot gate (#292) needs this: `unmanagedNames` suppresses + * only the ACTUAL side, which is right when the expected side is the metadata (it is + * already scoped) and wrong here, where the expected side IS the snapshot — a + * snapshot written before the scope was declared still carries the other owner's + * tables, and leaving them in reports a phantom disagreement about an object this + * consumer does not manage. + * + * `governed` is the scope decision the caller's drift comparison already made — pass + * the `DriftResult` itself, which satisfies this shape. Taking `declaredSchemas` + * from there rather than re-deriving it from the snapshot is what closes the last + * whole-database door: a snapshot that is present but EMPTY (a never-migrated + * project) declares no schemas at all, so deriving from it hands `diff` nothing and + * reaches its "no model, govern the whole database" fallback — the very inversion + * this module exists to prevent, at the one call site that was still re-deriving. + * + * An empty `outOfScope` returns the SAME snapshot object with no schema pin, so an + * unscoped project's `diff` arguments are byte-for-byte what they always were. + */ +export function excludeFromSnapshot( + snapshot: SchemaSnapshot, + governed: GovernedScope, +): ScopedExpectedSchema { + if (governed.outOfScope.length === 0) return { snapshot, outOfScope: [] }; + const excluded = new Set(governed.outOfScope); + const declared = governed.declaredSchemas ?? declaredSchemasOf(snapshot); + return { + snapshot: { + ...snapshot, + tables: splitOnName(snapshot.tables, excluded).rest, + views: splitOnName(snapshot.views, excluded).rest, + }, + outOfScope: [...governed.outOfScope], + declaredSchemas: [...declared], + }; +} + +/** The scope decision a run made, as `DriftResult` reports it. */ +export interface GovernedScope { + /** Qualified physical names (`.`) the run does not govern. */ + readonly outOfScope: readonly string[]; + /** The schemas the run governs — `ScopedExpectedSchema.declaredSchemas`. */ + readonly declaredSchemas?: readonly string[] | undefined; +} + +/** + * Partition `objs` on whether `qualifiedDbName(o)` is in `names`. + * + * `carryForwardOutOfScope` wants the `named` half (carry the excluded entries + * forward) and `excludeFromSnapshot` wants the `rest` half (drop them). They are + * exact complements over the same key function, so they share one traversal rather + * than two filters that could come to key differently. + */ +function splitOnName( + objs: readonly T[], + names: ReadonlySet, +): { named: T[]; rest: T[] } { + const named: T[] = []; + const rest: T[] = []; + for (const o of objs) (names.has(qualifiedDbName(o)) ? named : rest).push(o); + return { named, rest }; +} + +/** + * The three `diff` arguments a scoped run owes, as ONE value. + * + * The module header lists them as three separate obligations, and five call sites + * re-derived them by hand — one of which had already drifted into its own guard. + * Every scoped `diff` call is now + * `diff({ ...scopedDiffInputs(scoped, collectUnmanagedNames(metadata)), actual, ... })`, + * so the rule is enforced by the type rather than by the comment. + * + * `unmanaged` is the `@unmanaged`-declared set (`collectUnmanagedNames`); it is + * MERGED with `outOfScope`, never replaced by it — both must reach `diff`. + * `scopeSchemas` is omitted entirely when the run narrowed nothing, so an unscoped + * project's arguments are unchanged. + */ +export function scopedDiffInputs( + scoped: ScopedExpectedSchema, + unmanaged: readonly string[], +): Pick { + return { + expected: scoped.snapshot, + unmanagedNames: [...unmanaged, ...scoped.outOfScope], + ...(scoped.declaredSchemas !== undefined ? { scopeSchemas: scoped.declaredSchemas } : {}), + }; +} + +/** + * The distinct database schemas a snapshot's tables and views sit in, absent + * normalized to the Postgres default — the value `diff` derives for itself when no + * `scopeSchemas` is supplied. The ONE definition: any caller narrowing an expected + * side must pin `diff`'s schema scope to the UNNARROWED snapshot's schemas, and a + * second encoding of "absent means public" here would silently disagree with the + * one inside `diff`. + * + * Empty in ⇒ empty out, which callers translate to "pass nothing", preserving + * `diff`'s legacy whole-database fallback for a genuinely empty model. + */ +export function declaredSchemasOf(snapshot: SchemaSnapshot): string[] { + return [ + ...new Set( + [...snapshot.tables, ...snapshot.views].map( + (o) => o.schema ?? DEFAULT_DB_SCHEMA_POSTGRES, + ), + ), + ].sort(); +} + +/** + * Narrow an expected schema to the objects inside `inScope`. + * + * An undefined predicate returns the input untouched — the SAME snapshot object, + * not an equal copy — so a project that declares no `migrate.scope` reaches the + * diff, the emitter and the committed snapshot through an unchanged value. + * + * A table or view with NO recorded provenance is KEPT. Scope decides on the + * declaring object's FQN, and an object whose FQN is unknown was never proven to be + * anyone else's; dropping it would silently un-manage it (and, worse, suppressing + * its name on the actual side would hide real drift). + */ +export function scopeExpectedSchema( + built: ExpectedSchemaWithProvenance, + inScope: ObjectScopePredicate | undefined, +): ScopedExpectedSchema { + if (inScope === undefined) return { snapshot: built.snapshot, outOfScope: [] }; + + // Computed from `built.snapshot` — the UNSCOPED side — deliberately, and before + // the filter below runs. Deriving it from the survivors would reproduce exactly + // the defect this exists to close. + const declared = declaredSchemasOf(built.snapshot); + + const outOfScope: string[] = []; + const governed = (obj: T): boolean => { + const qualified = qualifiedDbName(obj); + const fqn = built.provenance.get(qualified); + if (fqn === undefined || inScope(fqn)) return true; + outOfScope.push(qualified); + return false; + }; + + return { + snapshot: { + ...built.snapshot, + tables: built.snapshot.tables.filter(governed), + views: built.snapshot.views.filter(governed), + }, + outOfScope, + ...(declared.length > 0 ? { declaredSchemas: declared } : {}), + }; +} diff --git a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts index f681d52e7..01ad1787d 100644 --- a/server/typescript/packages/migrate-ts/src/snapshot/plan.ts +++ b/server/typescript/packages/migrate-ts/src/snapshot/plan.ts @@ -1,8 +1,9 @@ // src/snapshot/plan.ts import type { ColumnNamingStrategy, MetaData } from "@metaobjectsdev/metadata"; -import { buildExpectedSchema } from "../expected-schema.js"; +import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../expected-schema.js"; import { diff, type DiffArgs } from "../diff/index.js"; import { collectUnmanagedNames } from "../unmanaged.js"; +import { carryForwardOutOfScope, scopeExpectedSchema, scopedDiffInputs, type ObjectScopePredicate } from "../scope.js"; import type { Dialect, DiffResult, SchemaSnapshot } from "../types.js"; import type { ExpectedViewInput } from "../expected-schema.js"; @@ -14,13 +15,34 @@ export interface PlanOfflineArgs extends Pick { - const nextSnapshot = buildExpectedSchema(args.metadata, { - dialect: args.dialect, - ...(args.columnNamingStrategy ? { columnNamingStrategy: args.columnNamingStrategy } : {}), - ...(args.views !== undefined ? { views: args.views } : {}), - }); + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(args.metadata, { + dialect: args.dialect, + ...(args.columnNamingStrategy ? { columnNamingStrategy: args.columnNamingStrategy } : {}), + ...(args.views !== undefined ? { views: args.views } : {}), + }), + args.inScope, + ); + // The DIFF runs against the narrowed side; the SNAPSHOT keeps what this run + // excluded. Committing the narrowed schema would delete every out-of-scope entry + // the previous snapshot held, so removing or widening `migrate.scope` later would + // propose CREATE TABLE for a table that exists and fail at apply. Byte-identical + // for an unscoped run (`outOfScope` empty ⇒ the same object). + const nextSnapshot = carryForwardOutOfScope(scoped.snapshot, args.snapshot, scoped.outOfScope); const result = await diff({ - expected: nextSnapshot, + // The three scoped-diff obligations as one value (see scope.ts's header). The + // `unmanagedNames` merge matters as much on the OFFLINE path as anywhere: an + // out-of-scope table already recorded in the snapshot must not be dropped just + // because the scope excludes it, and neither must a declared-@unmanaged one a + // `baseline --from-db` captured (#208 §7). + ...scopedDiffInputs(scoped, collectUnmanagedNames(args.metadata)), actual: args.snapshot, dialect: args.dialect, // #258 — migration generation refuses a primary-key MOVE (there is no primary-key // change kind to express it; it would otherwise silently drop the constraint). The // read-only verify/drift path does NOT set this, so `meta verify` still reports drift. refusePrimaryKeyChange: true, - // #208 §7 — exclude declared-@unmanaged objects from the actual (snapshot) side too, - // so the OFFLINE generate path never proposes DROP for an external table that a - // `baseline --from-db` captured into the snapshot (parity with the online/verify paths). - unmanagedNames: collectUnmanagedNames(args.metadata), ...(args.allow ? { allow: args.allow } : {}), ...(args.onAmbiguous ? { onAmbiguous: args.onAmbiguous } : {}), ...(args.ignoreTables ? { ignoreTables: args.ignoreTables } : {}), }); - return { diff: result, nextSnapshot }; + return { diff: result, nextSnapshot, expected: scoped.snapshot, outOfScope: scoped.outOfScope }; } /** Seed an initial reference snapshot from metadata (greenfield baseline). */ diff --git a/server/typescript/packages/migrate-ts/src/unmanaged.ts b/server/typescript/packages/migrate-ts/src/unmanaged.ts index 765f0dbc1..59c5892f5 100644 --- a/server/typescript/packages/migrate-ts/src/unmanaged.ts +++ b/server/typescript/packages/migrate-ts/src/unmanaged.ts @@ -10,10 +10,10 @@ import { isMetaSource, resolveTableSchema, - DEFAULT_DB_SCHEMA_POSTGRES, TYPE_OBJECT, type MetaData, } from "@metaobjectsdev/metadata"; +import { qualifiedDbName } from "./qualified-name.js"; /** * The qualified physical names (`schema.name`, schema defaulting to Postgres `public`) @@ -38,8 +38,7 @@ export function collectUnmanagedNames(root: MetaData): string[] { // make the class check false and silently un-silence a declared-@unmanaged // object, turning it back into a proposed drop. if (!isMetaSource(src) || !src.isUnmanaged) continue; - const schema = resolveTableSchema(obj) ?? DEFAULT_DB_SCHEMA_POSTGRES; - out.push(`${schema}.${src.physicalName}`); + out.push(qualifiedDbName({ name: src.physicalName, schema: resolveTableSchema(obj) })); } } return out; diff --git a/server/typescript/packages/migrate-ts/test/drift/drift-scope.test.ts b/server/typescript/packages/migrate-ts/test/drift/drift-scope.test.ts new file mode 100644 index 000000000..d6aa9249c --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/drift/drift-scope.test.ts @@ -0,0 +1,79 @@ +/** + * `meta verify --db` honours `migrate.scope` — the drift half. + * + * `computeDriftFromActual` is the single choke point for BOTH verify paths (the + * Kysely path and the D1 path), so the scope is threaded there. A table another + * owner's tool created is not this consumer's drift: it is reported as + * out-of-scope rather than as a change, and `verify` says how many were skipped + * (a table silently dropped from the comparison would read as a CHECKED table). + */ +import { describe, test, expect } from "bun:test"; +import { MetaDataLoader, InMemoryStringSource, type MetaRoot } from "@metaobjectsdev/metadata"; +import { buildExpectedSchema } from "../../src/expected-schema.js"; +import { computeDriftFromActual } from "../../src/drift/drift.js"; + +const META = JSON.stringify({ + "metadata.root": { + children: [ + { + "object.entity": { + name: "Job", + package: "acme::platform", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }, + { + "object.entity": { + name: "Match", + package: "arena", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"] } }, + ], + }, + }, + ], + }, +}); + +async function load(): Promise { + return (await new MetaDataLoader().load([new InMemoryStringSource(META)])).root; +} + +const platformOnly = (fqn: string): boolean => fqn.startsWith("acme::platform::"); + +describe("computeDriftFromActual — migrate.scope", () => { + test("an out-of-scope table present in the DB is NOT drift", async () => { + const root = await load(); + // The live database holds both tables; only `jobs` is this consumer's. + const actual = buildExpectedSchema(root, { dialect: "sqlite" }); + + const result = await computeDriftFromActual(actual, "sqlite", root, { inScope: platformOnly }); + expect(result.changes).toEqual([]); + expect(result.outOfScope).toEqual(["public.matches"]); + }); + + test("in-scope drift is still reported", async () => { + const root = await load(); + const actual = buildExpectedSchema(root, { dialect: "sqlite" }); + actual.tables = actual.tables.filter((t) => t.name !== "jobs"); + + const result = await computeDriftFromActual(actual, "sqlite", root, { inScope: platformOnly }); + expect(result.changes.map((c) => c.kind)).toContain("create-table"); + }); + + test("no scope → unchanged: every table is compared, nothing is out of scope", async () => { + const root = await load(); + const actual = buildExpectedSchema(root, { dialect: "sqlite" }); + actual.tables = actual.tables.filter((t) => t.name !== "matches"); + + const result = await computeDriftFromActual(actual, "sqlite", root); + expect(result.changes.map((c) => c.kind)).toEqual(["create-table"]); + expect(result.outOfScope).toEqual([]); + }); +}); diff --git a/server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts b/server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts new file mode 100644 index 000000000..f6cb48715 --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/expected-schema-scope.test.ts @@ -0,0 +1,322 @@ +/** + * Per-command scope (`migrate.scope`) — the expected-schema half. + * + * A consumer that owns one package tree's tables in a database another owner + * also writes to declares `migrate": { "scope": [...] }`. Tables outside that + * scope are neither created nor dropped, which takes TWO suppressions, not one: + * dropping them from the EXPECTED side alone would turn every out-of-scope + * table that exists in the database into a proposed DROP TABLE — the exact + * hazard the feature exists to remove. + * + * The scope decision is made on the DECLARING OBJECT's fully-qualified name, + * threaded out of the same Pass 1 walk that builds the tables (never re-derived + * from a SQL name, which is lossy, and never a second walk, which would drift + * from Pass 1's skip rules). + */ +import { describe, test, expect } from "bun:test"; +import { MetaDataLoader, InMemoryStringSource, type MetaRoot } from "@metaobjectsdev/metadata"; +import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../src/expected-schema.js"; +import { scopeExpectedSchema, scopedDiffInputs } from "../src/scope.js"; +import { diff } from "../src/diff/index.js"; +import { planOffline } from "../src/snapshot/plan.js"; +import { serializeSnapshot, SNAPSHOT_FORMAT_VERSION } from "../src/snapshot/serialize.js"; +import type { SchemaSnapshot } from "../src/types.js"; + +const PLATFORM = JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [ + { + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }, + ], + }, +}); + +const ARENA = JSON.stringify({ + "metadata.root": { + package: "arena", + children: [ + { + "object.entity": { + name: "Match", + children: [ + { "source.rdb": { name: "src", "@table": "matches" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }, + ], + }, +}); + +async function loadBoth(): Promise { + const loaded = await new MetaDataLoader().load([ + new InMemoryStringSource(PLATFORM), + new InMemoryStringSource(ARENA), + ]); + return loaded.root; +} + +/** The `migrate.scope: ["acme::platform::**"]` decision, without importing the + * pattern engine — migrate-ts takes a predicate precisely so it never carries a + * second implementation of one (`matchesScope` in the sdk is the only one). */ +const platformOnly = (fqn: string): boolean => fqn.startsWith("acme::platform::"); + +describe("scopeExpectedSchema", () => { + test("drops tables whose declaring object falls outside the scope", async () => { + const root = await loadBoth(); + const built = buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }); + expect(built.snapshot.tables.map((t) => t.name).sort()).toEqual(["jobs", "matches"]); + + const scoped = scopeExpectedSchema(built, platformOnly); + expect(scoped.snapshot.tables.map((t) => t.name)).toEqual(["jobs"]); + }); + + test("names the dropped tables so the ACTUAL side can be suppressed too", async () => { + const root = await loadBoth(); + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }), + platformOnly, + ); + // Qualified exactly as diff keys its tables (`.`, schema + // defaulting to the Postgres default), so the names feed `unmanagedNames`. + expect(scoped.outOfScope).toEqual(["public.matches"]); + }); + + test("an undefined scope leaves the expected schema untouched", async () => { + const root = await loadBoth(); + const built = buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }); + const scoped = scopeExpectedSchema(built, undefined); + // Same object, not merely an equal one: an unscoped project must reach the + // diff through byte-identical input. + expect(scoped.snapshot).toBe(built.snapshot); + expect(scoped.outOfScope).toEqual([]); + }); + + test("both sides: an out-of-scope table present in `actual` produces NO drop-table", async () => { + const root = await loadBoth(); + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }), + platformOnly, + ); + // The database holds BOTH tables — `matches` is another owner's, created by + // another tool. It is absent from the scoped expected side. + const actual: SchemaSnapshot = buildExpectedSchema(root, { dialect: "sqlite" }); + + const result = await diff({ + expected: scoped.snapshot, + actual, + dialect: "sqlite", + unmanagedNames: scoped.outOfScope, + }); + expect(result.changes).toEqual([]); + }); + + test("without the actual-side suppression the same diff WOULD drop it (the hazard is real)", async () => { + const root = await loadBoth(); + const scoped = scopeExpectedSchema( + buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }), + platformOnly, + ); + const actual: SchemaSnapshot = buildExpectedSchema(root, { dialect: "sqlite" }); + + const result = await diff({ expected: scoped.snapshot, actual, dialect: "sqlite" }); + expect(result.changes.map((c) => c.kind)).toEqual(["drop-table"]); + }); + + test("a table whose provenance is unknown is kept — scope never guesses", async () => { + const root = await loadBoth(); + const built = buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }); + const withStranger: typeof built = { + snapshot: { + ...built.snapshot, + tables: [ + ...built.snapshot.tables, + { name: "stranger", columns: [], indexes: [], foreignKeys: [], checks: [], primaryKey: [] }, + ], + }, + provenance: built.provenance, + }; + const scoped = scopeExpectedSchema(withStranger, platformOnly); + expect(scoped.snapshot.tables.map((t) => t.name)).toEqual(["jobs", "stranger"]); + expect(scoped.outOfScope).toEqual(["public.matches"]); + }); +}); + +/** + * A scope narrows which OBJECTS the run governs — never which SCHEMAS it may see. + * + * This is the consequence of pinning `scopeSchemas` to the UNSCOPED model, and it is + * easy to read the other way round, so it is pinned here rather than left in a review + * transcript. Excluding every declared object in a schema does NOT hand that schema + * over: it stays in scope, so an UNDECLARED table sitting in it is still a drop + * candidate — exactly as it would be on an unscoped run of the same model. + * + * The alternative (deriving the schema set from the survivors) reintroduces the + * inversion the pin exists to close: a scope matching nothing empties `expected`, + * `diff` reads that as "no model, govern the whole database", and every table in + * every schema becomes a drop candidate. Narrowing must never widen. + * + * The way to stop managing a schema is to remove its objects from the MODEL, or to + * declare them `@unmanaged` — both of which change what the model claims. A + * `migrate.scope` says who runs the migration, not what the model describes. + */ +const REPORTING = JSON.stringify({ + "metadata.root": { + package: "arena", + children: [ + { + "object.entity": { + name: "Standing", + children: [ + { "source.rdb": { name: "src", "@table": "standings", "@schema": "reporting" } }, + { "field.long": { name: "id" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }, + ], + }, +}); + +describe("scope narrows objects, never schemas", () => { + test("a schema whose every declared object is excluded STAYS in scope", async () => { + const loaded = await new MetaDataLoader().load([ + new InMemoryStringSource(PLATFORM), + new InMemoryStringSource(REPORTING), + ]); + const built = buildExpectedSchemaWithProvenance(loaded.root, { dialect: "postgres" }); + const scoped = scopeExpectedSchema(built, platformOnly); + + // `reporting` lost its only declared object, and is still pinned. + expect(scoped.snapshot.tables.map((t) => t.name)).toEqual(["jobs"]); + expect(scoped.declaredSchemas).toEqual(["public", "reporting"]); + + // A table nobody declared, living in that schema. It has no provenance, so it + // never reaches `outOfScope` and nothing suppresses it on the actual side. + const actual: SchemaSnapshot = { + tables: [ + ...built.snapshot.tables, + { name: "legacy_stats", schema: "reporting", columns: [], indexes: [], foreignKeys: [], checks: [], primaryKey: [] }, + ], + views: [], + }; + const result = await diff({ + ...scopedDiffInputs(scoped, []), + actual, + dialect: "postgres", + allow: { dropTable: true }, + }); + + // It IS a drop candidate — the same verdict an unscoped run of this model gives. + const drops = result.changes.filter((c) => c.kind === "drop-table"); + expect(drops).toHaveLength(1); + if (drops[0]?.kind !== "drop-table") throw new Error("expected a drop-table"); + expect(drops[0].table).toBe("legacy_stats"); + }); + + test("a schema the model never declares at all is untouched (the pin is not a widening)", async () => { + const loaded = await new MetaDataLoader().load([ + new InMemoryStringSource(PLATFORM), + new InMemoryStringSource(REPORTING), + ]); + const built = buildExpectedSchemaWithProvenance(loaded.root, { dialect: "postgres" }); + const scoped = scopeExpectedSchema(built, platformOnly); + + const actual: SchemaSnapshot = { + tables: [ + ...built.snapshot.tables, + { name: "events", schema: "analytics", columns: [], indexes: [], foreignKeys: [], checks: [], primaryKey: [] }, + ], + views: [], + }; + const result = await diff({ + ...scopedDiffInputs(scoped, []), + actual, + dialect: "postgres", + allow: { dropTable: true }, + }); + expect(result.changes).toEqual([]); + }); +}); + +describe("an accepted scoped run keeps out-of-scope entries in the committed snapshot", () => { + test("the snapshot planOffline hands back retains the excluded table", async () => { + const root = await loadBoth(); + const prior = buildExpectedSchema(root, { dialect: "sqlite" }); + expect(prior.tables.map((t) => t.name).sort()).toEqual(["jobs", "matches"]); + + const plan = await planOffline({ + metadata: root, + dialect: "sqlite", + snapshot: prior, + inScope: platformOnly, + }); + + // The DIFF side is narrowed — the run governs `jobs` only... + expect(plan.expected.tables.map((t) => t.name)).toEqual(["jobs"]); + // ...but the snapshot it commits still holds `matches`. Committing the narrowed + // schema would delete it, and removing `migrate.scope` later would then propose + // CREATE TABLE for a table that exists — a migration that fails at apply. + expect(plan.nextSnapshot.tables.map((t) => t.name).sort()).toEqual(["jobs", "matches"]); + }); + + test("an unscoped run commits the SAME object — byte-identical snapshot", async () => { + const root = await loadBoth(); + const prior = buildExpectedSchema(root, { dialect: "sqlite" }); + const plan = await planOffline({ metadata: root, dialect: "sqlite", snapshot: prior }); + expect(plan.nextSnapshot).toBe(plan.expected); + }); + + test("removing the scope after an accepted run proposes nothing (the round trip)", async () => { + const root = await loadBoth(); + const prior = buildExpectedSchema(root, { dialect: "sqlite" }); + + const scopedRun = await planOffline({ + metadata: root, dialect: "sqlite", snapshot: prior, inScope: platformOnly, + }); + // Second run, scope removed, diffing against what the first run committed. + const unscopedRun = await planOffline({ + metadata: root, dialect: "sqlite", snapshot: scopedRun.nextSnapshot, + }); + expect(unscopedRun.diff.changes).toEqual([]); + }); +}); + +describe("provenance never reaches the committed snapshot", () => { + test("a view's declaring FQN is recorded but never serialized; formatVersion stays 3", async () => { + const root = await loadBoth(); + const view = { name: "v_jobs", sql: "SELECT id FROM jobs", dependsOn: ["jobs"] }; + + const withFqn = buildExpectedSchemaWithProvenance(root, { + dialect: "sqlite", + views: [{ ...view, fqn: "acme::platform::JobSummary" }], + }); + const withoutFqn = buildExpectedSchema(root, { dialect: "sqlite", views: [view] }); + + // The provenance thread is invisible on disk: carrying it changes no bytes. + expect(serializeSnapshot(withFqn.snapshot)).toBe(serializeSnapshot(withoutFqn)); + expect(serializeSnapshot(withFqn.snapshot)).not.toContain("fqn"); + expect(SNAPSHOT_FORMAT_VERSION).toBe(3); + + // ...and it IS recorded, keyed like every other qualified name. + expect(withFqn.provenance.get("public.v_jobs")).toBe("acme::platform::JobSummary"); + }); + + test("a table's provenance is its declaring entity's resolutionKey()", async () => { + const root = await loadBoth(); + const { provenance } = buildExpectedSchemaWithProvenance(root, { dialect: "sqlite" }); + expect(provenance.get("public.jobs")).toBe("acme::platform::Job"); + expect(provenance.get("public.matches")).toBe("arena::Match"); + }); +}); diff --git a/server/typescript/packages/migrate-ts/test/scope-empty-match.test.ts b/server/typescript/packages/migrate-ts/test/scope-empty-match.test.ts new file mode 100644 index 000000000..210cd8dae --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/scope-empty-match.test.ts @@ -0,0 +1,135 @@ +/** + * Per-command scope (`migrate.scope`) — the EMPTY-MATCH case. + * + * Narrowing the expected side must never WIDEN what the diff governs. It could, + * through one mechanism: `diff` derives its schema scope from the schemas the + * EXPECTED side mentions, and falls back to "no schema scoping at all" when + * expected is empty (the legacy whole-DB path for a project with no model). A + * scope matching nothing empties `expected.tables`, hits that fallback, and every + * actual table in every schema becomes a drop candidate — including another + * owner's, which was never in `expected`, so it carries no provenance and is not + * in `outOfScope` either. + * + * So the declaration whose entire purpose is to stop migrate touching another + * owner's tables would CAUSE migrate to propose dropping one, whenever the scope + * is wrong (a typo'd or stale package pattern), silently. + * + * The fix is structural: `scopeExpectedSchema` reports the schemas the UNSCOPED + * model declares, and every caller threads that into `diff`'s `scopeSchemas`, so + * the schema scope is a property of the whole model and `migrate.scope` cannot + * move it in either direction. An unscoped project is untouched — no scope, no + * `declaredSchemas`, and `diff` derives its own exactly as before. + */ +import { describe, test, expect } from "bun:test"; +import { MetaDataLoader, InMemoryStringSource, type MetaRoot } from "@metaobjectsdev/metadata"; +import { buildExpectedSchema, buildExpectedSchemaWithProvenance } from "../src/expected-schema.js"; +import { scopeExpectedSchema } from "../src/scope.js"; +import { planOffline } from "../src/snapshot/plan.js"; +import { computeDriftFromActual } from "../src/drift/drift.js"; +import type { Change, SchemaSnapshot, TableDescriptor } from "../src/types.js"; + +/** One entity, in its own database schema — so "the schemas the model declares" + * is a proper subset of what the database holds. */ +const PLATFORM = JSON.stringify({ + "metadata.root": { + package: "acme::platform", + children: [ + { + "object.entity": { + name: "Job", + children: [ + { "source.rdb": { name: "src", "@table": "jobs", "@schema": "acme" } }, + { "field.long": { name: "id" } }, + { "field.string": { name: "title" } }, + { "identity.primary": { name: "pk", "@fields": ["id"], "@generation": "increment" } }, + ], + }, + }, + ], + }, +}); + +async function loadPlatform(): Promise { + const loaded = await new MetaDataLoader().load([new InMemoryStringSource(PLATFORM)]); + expect(loaded.errors).toHaveLength(0); + return loaded.root; +} + +/** Another owner's table, in the default schema this model never mentions. */ +const OTHER_APP_TABLE: TableDescriptor = { + name: "other_app_table", + schema: "public", + columns: [{ name: "id", sqlType: { kind: "integer", bits: 64 }, nullable: false }], + indexes: [], + foreignKeys: [], + checks: [], + primaryKey: ["id"], +}; + +/** The live database: this consumer's table exactly as declared, plus a table + * belonging to someone else entirely. */ +async function actualSchema(root: MetaRoot): Promise { + const mine = buildExpectedSchema(root, { dialect: "postgres" }); + return { ...mine, tables: [...mine.tables, OTHER_APP_TABLE] }; +} + +/** A `migrate.scope` that matches nothing — a typo'd or stale package pattern. */ +const matchesNothing = (): boolean => false; + +/** The proposed `DROP TABLE` names, by table name — the whole point of the probe. + * Narrowed on the discriminant so `table` is the string arm, never a descriptor. */ +const dropped = (changes: readonly Change[]): string[] => + changes.filter((c) => c.kind === "drop-table").map((c) => c.table); + +describe("migrate.scope matching nothing", () => { + test("CONTROL — with no scope at all, another owner's table in an undeclared schema is left alone", async () => { + const root = await loadPlatform(); + const plan = await planOffline({ + metadata: root, + dialect: "postgres", + snapshot: await actualSchema(root), + allow: { dropTable: true }, + }); + expect(dropped(plan.diff.changes)).toEqual([]); + }); + + test("a scope that matches nothing must NOT propose dropping it either", async () => { + const root = await loadPlatform(); + const plan = await planOffline({ + metadata: root, + dialect: "postgres", + snapshot: await actualSchema(root), + inScope: matchesNothing, + allow: { dropTable: true }, + }); + // The consumer's own table left the expected side, as declared... + expect(plan.outOfScope).toEqual(["acme.jobs"]); + // ...and nothing at all is proposed for the schema the model never mentions. + expect(dropped(plan.diff.changes)).toEqual([]); + }); + + test("verify --db sees the same thing — no phantom drift for another owner's table", async () => { + const root = await loadPlatform(); + const drift = await computeDriftFromActual( + await actualSchema(root), + "postgres", + root, + { inScope: matchesNothing }, + ); + expect(drift.outOfScope).toEqual(["acme.jobs"]); + expect(drift.changes).toEqual([]); + }); + + test("the schema scope reported is the UNSCOPED model's, so narrowing can never widen it", async () => { + const root = await loadPlatform(); + const built = buildExpectedSchemaWithProvenance(root, { dialect: "postgres" }); + + // Unscoped: nothing reported, so `diff` derives its own exactly as before — + // an unscoped project reaches the diff through byte-identical arguments. + expect(scopeExpectedSchema(built, undefined).declaredSchemas).toBeUndefined(); + + // Scoped: the full model's schemas, not the survivors'. + expect(scopeExpectedSchema(built, matchesNothing).declaredSchemas).toEqual(["acme"]); + expect(scopeExpectedSchema(built, () => true).declaredSchemas).toEqual(["acme"]); + }); +}); diff --git a/server/typescript/packages/migrate-ts/test/scope-snapshot-gate.test.ts b/server/typescript/packages/migrate-ts/test/scope-snapshot-gate.test.ts new file mode 100644 index 000000000..8f3e38c67 --- /dev/null +++ b/server/typescript/packages/migrate-ts/test/scope-snapshot-gate.test.ts @@ -0,0 +1,94 @@ +/** + * `excludeFromSnapshot` — the scoped-diff door for a COMMITTED SNAPSHOT. + * + * `verify`'s committed-snapshot gate (#292) runs a SECOND comparison over the same + * run: the committed `.schema..json` against the live database. It owes the + * same three obligations as every other scoped diff (migrate-ts `scope.ts` header), + * and it was the one call site re-deriving them by hand — including the schema pin, + * which it derived from the SNAPSHOT rather than from the model. + * + * That re-derivation left one door open: a snapshot that is present but EMPTY (a + * never-migrated project) declares no schemas at all, so the pin came out empty, the + * caller's `length > 0` guard dropped it, and `diff` fell back to "no model, govern + * the whole database" — reporting every table another owner has, in schemas this + * model never mentions, as a snapshot disagreement. + */ +import { describe, test, expect } from "bun:test"; +import { diff } from "../src/diff/index.js"; +import { excludeFromSnapshot, scopedDiffInputs } from "../src/scope.js"; +import type { SchemaSnapshot, TableDescriptor } from "../src/types.js"; + +function table(name: string, schema: string): TableDescriptor { + return { + name, + schema, + columns: [ + { name: "id", sqlType: { kind: "integer", bits: 64 }, nullable: false, identity: "increment" }, + ], + indexes: [], + foreignKeys: [], + primaryKey: ["id"], + checks: [], + }; +} + +/** A never-migrated committed snapshot: the file exists, it records nothing. */ +const EMPTY_SNAPSHOT: SchemaSnapshot = { tables: [], views: [] }; + +/** The live database: this consumer's table, the co-owner's table in the SAME + * schema (declared by the model, excluded by `migrate.scope`), and a third + * party's table in a schema this model never mentions at all. */ +const ACTUAL: SchemaSnapshot = { + tables: [table("jobs", "public"), table("matches", "public"), table("events", "analytics")], + views: [], +}; + +/** The scope decision the drift comparison already made for this run: `matches` is + * another owner's, and the model declares into `public` only. */ +const GOVERNED = { outOfScope: ["public.matches"], declaredSchemas: ["public"] }; + +describe("excludeFromSnapshot", () => { + test("an EMPTY committed snapshot under a scope does not govern the whole database", async () => { + const scoped = excludeFromSnapshot(EMPTY_SNAPSHOT, GOVERNED); + const result = await diff({ + ...scopedDiffInputs(scoped, []), + actual: ACTUAL, + allow: {}, + }); + + // analytics.events is a third party's, in a schema this model never declares. + // It has no provenance, so it can never reach `outOfScope` — the SCHEMA pin is + // the only thing that can protect it, and re-deriving that pin from the empty + // snapshot is what handed `diff` the whole database instead. + expect(result.changes.map((c) => JSON.stringify(c)).join("\n")).not.toContain("events"); + // The governed table is still compared: an empty snapshot really does disagree + // with a database that holds `public.jobs`, and that finding must survive. + expect(result.changes.filter((c) => c.kind === "drop-table")).toHaveLength(1); + // …and the co-owner's in-schema table is suppressed through `unmanagedNames`. + expect(result.changes.map((c) => JSON.stringify(c)).join("\n")).not.toContain("matches"); + }); + + test("nothing out of scope returns the SAME snapshot object and no schema pin", () => { + const scoped = excludeFromSnapshot(EMPTY_SNAPSHOT, { outOfScope: [], declaredSchemas: ["public"] }); + // Identity, not equality: an unscoped run's `diff` arguments must be exactly + // what they were before scope existed. + expect(scoped.snapshot).toBe(EMPTY_SNAPSHOT); + expect(scoped.declaredSchemas).toBeUndefined(); + expect(scopedDiffInputs(scoped, ["public.legacy"])).toEqual({ + expected: EMPTY_SNAPSHOT, + unmanagedNames: ["public.legacy"], + }); + }); + + test("out-of-scope entries leave the snapshot's own tables and views", () => { + const snapshot: SchemaSnapshot = { + tables: [table("jobs", "public"), table("matches", "public")], + views: [], + }; + const scoped = excludeFromSnapshot(snapshot, GOVERNED); + expect(scoped.snapshot.tables.map((t) => t.name)).toEqual(["jobs"]); + // The pin is the schemas the RUN governs, which is a property of the model — + // not of whatever survived the filter. + expect(scoped.declaredSchemas).toEqual(["public"]); + }); +}); diff --git a/server/typescript/packages/sdk/src/collection.ts b/server/typescript/packages/sdk/src/collection.ts new file mode 100644 index 000000000..5b915cd6e --- /dev/null +++ b/server/typescript/packages/sdk/src/collection.ts @@ -0,0 +1,183 @@ +// server/typescript/packages/sdk/src/collection.ts +// +// Phase-1 metadata-source-resolution — the single authority. +// +// `resolveCollection()` composes discovery (`discovery.ts`), config +// (`config.ts`), source resolution (`sources.ts`) and the scope engine +// (`scope.ts`) into one function that decides where a project's metadata +// lives. `metaobjects/` is the DEFAULT value of `sources`, never a +// requirement — a project that declares nothing still resolves exactly as +// today (`DEFAULT_SOURCES` in `sources.ts`); a project that declares +// `sources` can point anywhere. No other call site may assume the directory +// name — this is where that assumption is allowed to live, exactly once. +import { join, resolve } from "node:path"; +import { ParseError, codeSource } from "@metaobjectsdev/metadata"; +import { CONFIG_FILE, loadConfig, type Config } from "./config.js"; +import { discoverCollectionRoot, exists, isDir } from "./discovery.js"; +import { compileScope, matchesScope, type Scope } from "./scope.js"; +import { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./metadata-files.js"; +import { + DEFAULT_SOURCES, + orderedPathSpecs, + resolveSpecPath, + resolveSources, + type ResolvedSource, + type SourceSpec, +} from "./sources.js"; + +export interface Collection { + /** Directory whose config declared this collection (or the resolved start + * directory, when nothing was discovered and the default applies). */ + readonly configDir: string; + /** Canonically-ordered absolute metadata file paths — see `resolveSources`. + * Canonical, not sorted: within a directory source the walk order the + * toolchain has always used is preserved, because it survives into + * generated output. */ + readonly files: readonly string[]; + /** Same set, carrying the contributing spec for provenance. */ + readonly sources: readonly ResolvedSource[]; + /** The distinct roots the declared source specs resolve to, absolute, in the + * same canonical (content) order `files` uses. Derived from the DECLARED + * specs, not from the resolved files, so a source directory that legitimately + * holds no metadata still appears — a consumer listing "where this model + * comes from" (`meta docs --site` groups its pages by source root) must not + * silently lose a declared source because it happens to be empty today. */ + readonly sourceRoots: readonly string[]; + /** + * Output filter for codegen: does this fully-qualified name survive the + * collection's `scope`? Always defined — an unconfigured project compiles to + * an empty include/exclude, which admits everything, so callers pass this + * through unconditionally rather than branching. + * + * A PREDICATE rather than the `CompiledScope` it closes over, because nothing + * consumes a compiled scope as a compiled scope: every consumer immediately + * wrapped it in exactly this lambda, and `migrateScopePatterns` exists + * precisely because the compiled form cannot be shown to a human. + * `compileScope`/`matchesScope` stay exported for the conformance corpus. + */ + readonly inScope: (fqn: string) => boolean; + /** Output filter for migrate/verify --db (`migrate.scope`). Undefined => the + * command governs everything loaded, and that undefined is load-bearing: it + * is what leaves the expected schema untouched (migrate-ts `scope.ts`). */ + readonly inMigrateScope: ((fqn: string) => boolean) | undefined; + /** The patterns `inMigrateScope` was compiled FROM, for diagnostics only — + * `compileScope` produces RegExps, and a regex source is not something to + * show an author who wrote `acme::platform::**`. Carried so the "your scope + * matched nothing" refusal can name the patterns that missed. Always in + * lockstep with `inMigrateScope`: both undefined, or both present. */ + readonly migrateScopePatterns: readonly string[] | undefined; +} + +/** Narrow the zod-inferred `Config["scope"]` (whose `.optional()` fields are + * typed `T | undefined` even when present) down to `Scope`'s + * exactOptionalPropertyTypes-safe shape — a key is omitted entirely rather + * than assigned `undefined`. */ +function toScope(spec: Config["scope"]): Scope { + return { + ...(spec?.include !== undefined && { include: spec.include }), + ...(spec?.exclude !== undefined && { exclude: spec.exclude }), + }; +} + +/** + * THE single authority on where metadata lives. Every read path routes + * through this — `metaobjects/` is the DEFAULT value of `sources`, never an + * assumption baked into a call site. + * + * Resolution order: an explicit `opts.explicitDir` wins outright; otherwise + * `discoverCollectionRoot` walks up from `startDir` for the nearest directory + * carrying `.metaobjects/config.json` — the ONLY project marker (`discovery.ts` + * says why a directory that merely holds metadata is not one) — falling back to + * `startDir` itself when none is found. When the resolved directory carries a + * config, its declared `sources`/`scope`/`migrate.scope` govern. Only a + * genuinely ABSENT `config.json` falls through to `DEFAULT_SOURCES` — the same + * directory the pre-source-resolution toolchain always read; a config.json + * that EXISTS but fails to load (malformed JSON, schema violation) is the + * author's error and propagates rather than silently degrading — a source + * that fails to resolve must never look like one that was never declared. + * Throws `ERR_COLLECTION_NOT_FOUND` only when BOTH have failed: no + * `sources` were declared AND the default source directory does not exist + * either. + * + * A declared source that fails to resolve is a different, louder failure — + * `resolveSources` throws `ERR_SOURCE_UNRESOLVED` for that case; only the + * DEFAULT is allowed to be silently absent. + */ +export async function resolveCollection( + startDir: string, + opts?: { explicitDir?: string }, +): Promise { + const explicit = opts?.explicitDir; + + // Whether `configDir` carries a `config.json` — threaded through rather + // than re-`stat`'d below. On the non-explicit path, `discoverCollectionRoot` + // already proved it either way: it reports `hasConfig` from the same + // `.metaobjects/config.json` probe that decided where to stop, and reports + // false only after confirming that file is absent at every directory it + // examined, `resolve(startDir)` included. A second `stat` of the identical + // file would just re-prove what discovery established. The check is only + // load-bearing on the `explicitDir` path, where discovery never runs at all. + let configDir: string; + let hasConfig: boolean; + if (explicit !== undefined) { + configDir = resolve(explicit); + hasConfig = await exists(join(configDir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE)); + } else { + ({ dir: configDir, hasConfig } = await discoverCollectionRoot(startDir)); + } + + let specs: readonly SourceSpec[] = DEFAULT_SOURCES; + let scopeSpec: Config["scope"]; + let migrateSpec: string[] | undefined; + + if (hasConfig) { + // No try/catch here: a config.json that EXISTS but fails to load + // (malformed JSON, a ConfigSchema violation) propagates. Swallowing it + // would make a typo'd config behave identically to no config at all — + // silently generating from a possibly-stale `metaobjects/` with no + // diagnostic, which is a worse failure than the one this design exists + // to remove. + const cfg = await loadConfig(join(configDir, DEFAULT_METAOBJECTS_DIR)); + if (cfg.sources.length > 0) specs = cfg.sources; + scopeSpec = cfg.scope; + migrateSpec = cfg.migrate?.scope; + } + + // Only the DEFAULT is allowed to be absent — an explicitly declared source + // that does not resolve is `resolveSources`'s ERR_SOURCE_UNRESOLVED, not this. + // + // This re-`stat`s a directory the non-explicit discovery walk may already + // have probed, and that redundancy is intentional: it exists to produce the + // friendlier `ERR_COLLECTION_NOT_FOUND` diagnostic below rather than the raw + // `ERR_SOURCE_UNRESOLVED` `resolveSources` would throw on a genuinely missing + // default directory. Trading that clearer error for one syscall is a bad + // trade. It is also load-bearing outright on the `explicitDir` path and + // whenever a discovered config declares no `sources`, where nothing has + // probed it at all. + if (specs === DEFAULT_SOURCES && !(await isDir(join(configDir, DEFAULT_METADATA_DIR)))) { + throw new ParseError( + `no metadata sources declared in ${configDir} and no default "${DEFAULT_METADATA_DIR}" directory found. ` + + `Declare "sources" in ${DEFAULT_METAOBJECTS_DIR}/config.json, or run 'meta init' to scaffold.`, + { code: "ERR_COLLECTION_NOT_FOUND", source: codeSource("resolveCollection") }, + ); + } + + const sources = await resolveSources(configDir, specs); + const scope = compileScope(toScope(scopeSpec)); + const migrateScope = + migrateSpec === undefined ? undefined : compileScope({ include: migrateSpec }); + return { + configDir, + files: sources.map((s) => s.file), + sources, + // Canonical (content) order, from `resolveSources`'s own ordering — so this + // list is a pure function of the source SET, exactly like `files`. + sourceRoots: [ + ...new Set(orderedPathSpecs(specs).map((spec) => resolveSpecPath(configDir, spec))), + ], + inScope: (fqn: string): boolean => matchesScope(fqn, scope), + inMigrateScope: + migrateScope === undefined ? undefined : (fqn: string): boolean => matchesScope(fqn, migrateScope), + migrateScopePatterns: migrateSpec, + }; +} diff --git a/server/typescript/packages/sdk/src/config.ts b/server/typescript/packages/sdk/src/config.ts index eaf548c0c..5f16a466e 100644 --- a/server/typescript/packages/sdk/src/config.ts +++ b/server/typescript/packages/sdk/src/config.ts @@ -1,6 +1,7 @@ import { z } from "zod"; import { readFile, writeFile } from "node:fs/promises"; import { join } from "node:path"; +import type { SourceSpec } from "./sources.js"; const DialectEnum = z.enum(["sqlite", "postgres", "d1"]); @@ -32,12 +33,15 @@ export const AllowTokenEnum = z.enum([ "drop-identity-default", ]); +// .strict(), like every other object in this schema: `.partial()` alone leaves +// zod's default STRIP policy in place, so a misspelled key is silently deleted +// and the block reads as if the author never wrote it. const D1Block = z.object({ binding: z.string(), remote: z.boolean(), autoApply: z.boolean(), wranglerConfigPath: z.string(), -}).partial(); +}).partial().strict(); /** #192 — migration output-format adapters; orthogonal to dialect. */ const MigrateFormatEnum = z.enum(["default", "flyway"]); @@ -50,8 +54,71 @@ const MigrateBlock = z.object({ onAmbiguous: OnAmbiguousEnum, allow: z.array(AllowTokenEnum), d1: D1Block, -}).partial(); + /** Restricts a `meta migrate` run to a subset of the loaded metadata, by the + * same package-glob pattern grammar as top-level `scope` (see `scope.ts`). + * Include-only — there's no `migrate.scope.exclude`, since a migration run + * is scoped to what it's touching, not filtered down from "everything". */ + scope: z.array(z.string().min(1)), +// .strict() for the same reason as the top level and the source arms below, and +// with sharper teeth here: `{ migrate: { scopee: [...], dialect: "postgres" } }` +// used to parse to `{ dialect: "postgres" }`, so a typo'd `scope` key meant +// "unscoped" — silently governing every table in the database, which is the +// hazard `migrate.scope` exists to remove. +}).partial().strict(); +/** + * Mirrors the hand-written `SourceSpec` union in `./sources.ts` — a + * declared source kind, one of `path` (resolves today), `resource`, or + * `package` (both reserved, throw `ERR_SOURCE_KIND_UNSUPPORTED` until a + * later phase). `.strict()` on every arm: this project is fail-closed on + * undeclared keys everywhere else (ADR-0023 makes an unregistered metadata + * attribute a hard error for the same reason) — a config schema that + * silently strips an unknown key would let `{ path: "model", pathh: "typo" + * }` parse clean and resolve one source instead of erroring on the typo. + * The pre-phase-1 `{ kind: "path", path: "..." }` shape (the dead 2-arm + * discriminated union this replaces) never shipped to an adopter — `meta + * init` has only ever scaffolded `"sources": []`, and nothing under `src/` + * ever read the old shape — so there is no live config to be lenient for; + * it only ever existed in this package's own tests, updated alongside this + * schema. + */ +const SourceSpecSchema = z.union([ + z.object({ path: z.string().min(1) }).strict(), + z.object({ resource: z.string().min(1) }).strict(), + z.object({ package: z.string().min(1) }).strict(), +]); + +// Compile-time parity, BOTH directions: if SourceSpecSchema and the +// hand-written SourceSpec (./sources.ts) ever drift, one of these two +// assignments stops compiling. Each direction alone catches only HALF the +// drift — `z.infer<...>` assignable to `SourceSpec` catches an arm added to +// the schema but missing from SourceSpec, while `SourceSpec` assignable to +// `z.infer<...>` catches the opposite: an arm added to the hand-written +// SourceSpec that the schema never gained. A single one-directional +// assignment (the prior form of this guard) let a SourceSpec-only addition +// compile clean — proven by deliberately breaking each direction in +// isolation; see the quality-pass report for both failing `tsc` outputs. A +// conditional type (`X extends Y ? true : never`) would silently resolve to +// `never` instead of erroring — this direct-assignment form fails for real. +const _sourceSpecParityInferToSpec: SourceSpec = {} as z.infer; +const _sourceSpecParitySpecToInfer: z.infer = {} as SourceSpec; +void _sourceSpecParityInferToSpec; +void _sourceSpecParitySpecToInfer; + +/** Mirrors the hand-written `Scope` interface in `./scope.ts`. An absent or + * empty `include` means "everything" — see `matchesScope`. */ +const ScopeSchema = z + .object({ + include: z.array(z.string().min(1)).optional(), + exclude: z.array(z.string().min(1)).optional(), + }) + .strict(); + +// .strict() at the TOP level too, not just the source-spec arms and the +// scope block: without it, a misspelled top-level key (e.g. "scopes" for +// "scope") is silently stripped by zod and the collection resolves as +// "everything in scope" — the exact silent fail-open .strict() on the +// nested arms exists to prevent, one level up. export const ConfigSchema = z.object({ schema_version: z.literal(1), pending_in_git: z.boolean().default(true), @@ -61,27 +128,27 @@ export const ConfigSchema = z.object({ drift_warn: z.number().min(0).max(1).default(0.7), }) .default({}), - sources: z - .array( - z.union([ - z.object({ kind: z.literal("path"), path: z.string() }), - z.object({ kind: z.literal("package"), package: z.string() }), - ]), - ) - .default([]), + sources: z.array(SourceSpecSchema).default([]), + /** Output filter applied across every command — see `./scope.ts`. Absent + * means "everything" (no filtering), matching `Scope`'s own contract. */ + scope: ScopeSchema.optional(), extract: z .object({ metaignore: z.string().optional(), }) .default({}), migrate: MigrateBlock.optional(), -}); +}).strict(); export type Config = z.infer; export const DEFAULT_CONFIG: Config = ConfigSchema.parse({ schema_version: 1 }); -const CONFIG_FILE = "config.json"; +/** `config.json`'s basename — the single owner. `discovery.ts` and + * `collection.ts` import this rather than each keeping their own copy of + * the literal (a duplication that had drifted into three separate + * declarations of the same string). */ +export const CONFIG_FILE = "config.json"; export async function loadConfig(metaRoot: string): Promise { const raw = await readFile(join(metaRoot, CONFIG_FILE), "utf8"); diff --git a/server/typescript/packages/sdk/src/discovery.ts b/server/typescript/packages/sdk/src/discovery.ts new file mode 100644 index 000000000..b66922945 --- /dev/null +++ b/server/typescript/packages/sdk/src/discovery.ts @@ -0,0 +1,110 @@ +// server/typescript/packages/sdk/src/discovery.ts +// +// Phase-1 metadata-source-resolution — nearest-ancestor collection discovery. +// +// Walks up from a starting directory to find the nearest directory that IS a +// project root. This is what makes a CLI *contextual*: run it inside an app in +// a monorepo and it finds that app's configuration rather than the repo root's. +// +// Three properties are load-bearing. +// +// 1. **One marker.** A directory is a project root when it carries +// `.metaobjects/config.json`, and on no other evidence. A directory that +// merely *holds* metadata is not a project boundary: where metadata lives is +// the `sources` key's answer, and `metaobjects/` is only that key's default +// value. Stopping on a bare `metaobjects/` directory would put a second +// definition of "where metadata lives" back into the walk — the exact +// duplication `resolveCollection` exists to be the only instance of — and it +// would ignore a project whose config points its `sources` somewhere else +// entirely. See design §4.6.1. +// 2. **Nearest wins** — the walk returns on the FIRST directory carrying the +// marker, so a config in a subdirectory beats one in an ancestor. +// 3. **The walk stops at a repository boundary** (`.git`), so a monorepo +// checkout can never silently adopt a *parent checkout's* configuration. The +// marker check runs BEFORE the `.git` check within each directory — +// reversed, a root-level project (where `.git` also lives) would be +// unreachable from any subdirectory, since the boundary would stop the walk +// one directory too early. +import { stat } from "node:fs/promises"; +import { dirname, join, resolve } from "node:path"; +import { CONFIG_FILE } from "./config.js"; +import { DEFAULT_METAOBJECTS_DIR } from "./metadata-files.js"; + +const GIT_DIR = ".git"; + +/** Exported for reuse — `collection.ts` had its own byte-identical copy + * (`fileExists`); one definition, imported. */ +export async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +/** `exists`, narrowed to directories — `collection.ts`'s default-source probe + * needs that distinction to raise its friendlier `ERR_COLLECTION_NOT_FOUND` + * (a plain FILE where the default source directory should be is not a + * metadata home). Lives here beside `exists` so there is one filesystem + * predicate pair in the package rather than a copy per caller. */ +export async function isDir(path: string): Promise { + try { + return (await stat(path)).isDirectory(); + } catch { + return false; + } +} + +/** Where the discovery walk stopped, and what it found there. */ +export interface DiscoveredRoot { + /** The project root the walk settled on. Always absolute; falls back to the + * resolved start directory when the walk found no marker at all. */ + readonly dir: string; + /** Whether `dir` carries `.metaobjects/config.json`. False only on the + * no-marker fallback, where the DEFAULT sources apply. */ + readonly hasConfig: boolean; +} + +/** + * Walk up from `startDir` for the nearest project root — a directory holding + * `.metaobjects/config.json`, and nothing else (see the file header). The walk + * stops after examining a directory that contains `.git`, so a monorepo can + * never silently adopt a parent checkout's configuration. + * + * Never fails: with no config anywhere below the boundary it reports the + * resolved `startDir` with `hasConfig: false`, which is what + * `resolveCollection` turns into either the default source or + * `ERR_COLLECTION_NOT_FOUND`. + */ +export async function discoverCollectionRoot(startDir: string): Promise { + const start = resolve(startDir); + let dir = start; + for (;;) { + if (await exists(join(dir, DEFAULT_METAOBJECTS_DIR, CONFIG_FILE))) { + return { dir, hasConfig: true }; + } + // Boundary check AFTER the marker check: a repo-root project (sharing its + // directory with `.git`) is still findable from any subdirectory. + if (await exists(join(dir, GIT_DIR))) break; + const parent = dirname(dir); + if (parent === dir) break; + dir = parent; + } + return { dir: start, hasConfig: false }; +} + +/** + * The directory whose configuration governs a run started in `startDir` — the + * `dir` half of {@link discoverCollectionRoot}. + * + * Exists as its own export for the callers that must NOT require metadata to + * exist: `meta migrate apply-pending` and `--rollback` replay committed SQL and + * load no model at all, so they resolve their `.metaobjects/` directory through + * this rather than through `resolveCollection`. Sharing the walk is the point — + * a second "find the project root" implementation is how the migrations + * directory and the metadata directory come to disagree. + */ +export async function resolveConfigDir(startDir: string): Promise { + return (await discoverCollectionRoot(startDir)).dir; +} diff --git a/server/typescript/packages/sdk/src/index.ts b/server/typescript/packages/sdk/src/index.ts index 3c87b14d9..7498bb274 100644 --- a/server/typescript/packages/sdk/src/index.ts +++ b/server/typescript/packages/sdk/src/index.ts @@ -76,17 +76,34 @@ export { } from "./forge-types.js"; export type { ForgeType, ForgeAttr } from "./forge-types.js"; -// Memory loader — read metaobjects/ into a MetaData tree -// (workspace-aware: walks extends: deps via pnpm-workspace.yaml or -// package.json workspaces field if present) -export { - loadMemory, - defaultLoadMemoryProviders, - DEFAULT_METADATA_DIR, - DEFAULT_METAOBJECTS_DIR, -} from "./memory.js"; +// Memory loader — read a project's resolved metadata into a MetaData tree. +// Where those files come from is `resolveCollection`'s decision (below), which +// `loadMemory` calls when the caller supplies no explicit file set. +export { loadMemory, defaultLoadMemoryProviders } from "./memory.js"; export type { LoadMemoryOptions } from "./memory.js"; +// Default project layout — the DEFAULT value of `sources` (applied by +// `resolveCollection` alone) and the fixed directory holding the config that +// declares them. Exported for `meta init`, which SCAFFOLDS that layout. +export { DEFAULT_METADATA_DIR, DEFAULT_METAOBJECTS_DIR } from "./metadata-files.js"; + +// Scope — output filter over fully-qualified node names +export { compileScope, matchesScope } from "./scope.js"; +export type { Scope, CompiledScope } from "./scope.js"; + +// Source resolution — a declared source SET to a canonically-sorted file list +export { resolveSources, resolveSpecPath, orderedPathSpecs, DEFAULT_SOURCES } from "./sources.js"; +export type { SourceSpec, ResolvedSource } from "./sources.js"; + +// Discovery — nearest-ancestor project root (a `.metaobjects/config.json`, +// the only marker), bounded by the repo root +export { discoverCollectionRoot, resolveConfigDir } from "./discovery.js"; +export type { DiscoveredRoot } from "./discovery.js"; + +// Collection — the single authority on where a project's metadata lives +export { resolveCollection } from "./collection.js"; +export type { Collection } from "./collection.js"; + // Workspace discovery — finds peer metadata packages in a monorepo export { discoverWorkspace, resolveExtendsOrder, packageLabel } from "./workspace.js"; export type { Workspace, WorkspacePackage } from "./workspace.js"; diff --git a/server/typescript/packages/sdk/src/memory.ts b/server/typescript/packages/sdk/src/memory.ts index 860c4d5b4..796bdf04b 100644 --- a/server/typescript/packages/sdk/src/memory.ts +++ b/server/typescript/packages/sdk/src/memory.ts @@ -1,5 +1,3 @@ -import { join } from "node:path"; -import { readdir, stat } from "node:fs/promises"; import { composeRegistry, coreProviders, @@ -8,21 +6,8 @@ import { type MetaRoot, } from "@metaobjectsdev/metadata"; import { FileSource } from "@metaobjectsdev/metadata/core"; +import { resolveCollection } from "./collection.js"; import { forgeTypesProvider } from "./forge-types.js"; -import { discoverWorkspace, resolveExtendsOrder } from "./workspace.js"; - -/** - * Default directory name (relative to project root) where metadata JSON files - * are scanned. Scaffold via `meta init`; the directory is committed to git. - */ -export const DEFAULT_METADATA_DIR = "metaobjects"; - -/** - * Default directory name (relative to project root) for MetaObjects' own - * runtime state: config.json, .gen-state/, package.meta.json, agent docs. - * Scaffold via `meta init`; most contents are committed to git. - */ -export const DEFAULT_METAOBJECTS_DIR = ".metaobjects"; /** * Options for {@link loadMemory}. Consumers can supply additional @@ -50,6 +35,19 @@ export interface LoadMemoryOptions { * the `meta verify` command opts in to `true` (strict-by-default, #96). */ strict?: boolean; + /** + * An already-resolved, absolute metadata file list — normally + * `resolveCollection(...).files`. When supplied, `loadMemory` loads exactly + * these files and resolves nothing itself. + * + * Omitting it is not a different WAY of finding metadata, only a different + * place the same resolution happens: `loadMemory` then calls + * `resolveCollection(repoRoot)` itself. Passing it saves the second + * resolution when the caller already holds a collection (every routed CLI + * command does) and lets a caller load a file set it computed some other + * way; it can no longer diverge from what the config declares. + */ + files?: readonly string[]; } /** Default provider bundle threaded by {@link loadMemory} when no options @@ -62,10 +60,15 @@ export const defaultLoadMemoryProviders: readonly MetaDataTypeProvider[] = [ ]; /** - * Load all metadata files from `/metaobjects/` into a single - * MetaData. If `/.meta/package.meta.json` declares `extends:` deps - * and a workspace can be discovered (pnpm-workspace.yaml or package.json - * workspaces), peer packages are loaded too in topological dep-first order. + * Load a project's metadata into a single MetaData tree. + * + * Which files those are is `resolveCollection`'s decision, never this + * function's: with no {@link LoadMemoryOptions.files} it calls + * `resolveCollection(repoRoot)` — nearest-ancestor `.metaobjects/config.json`, + * then that config's declared `sources`, falling back to the default source + * directory only when a project declares none. `loadMemory` names no directory + * of its own, so a caller cannot end up loading from somewhere the rest of the + * toolchain does not. * * Excludes `_pending/`. Registers metaobjects core types plus Meta Forge's * descriptive top-level types (decision, principle, etc.) so mixed content @@ -73,11 +76,15 @@ export const defaultLoadMemoryProviders: readonly MetaDataTypeProvider[] = [ * {@link LoadMemoryOptions.providers}) are composed AFTER the defaults so * they may depend on core/forge ids. * - * Throws if `metaobjects/` doesn't exist (callers should run `meta init`). + * Throws `ERR_COLLECTION_NOT_FOUND` when nothing resolves (callers should run + * `meta init`), unless `options.files` is supplied. * - * @param repoRoot The project's working-directory root (e.g. process.cwd()). - * `loadMemory` resolves `metaobjects/` and (if workspace-aware) the - * transitive `extends:` graph automatically. + * @param repoRoot Where resolution STARTS — the working directory, typically + * `process.cwd()`. The walk goes up from here for the governing config, so + * this need not be the project root itself. + * **Ignored entirely when `options.files` is supplied**: that list is already + * resolved, so nothing reads this path. Every routed CLI command passes both, + * and the argument is inert at all of them. * @param options Optional {@link LoadMemoryOptions} — supply additional * providers or replace the default bundle entirely. */ @@ -99,10 +106,15 @@ export async function loadMemory( } const registry = composeRegistry(providers); - // Collect all metadata file paths to load. Order matters for the parser's - // deferred-resolution pass (it parses in array order, then resolves supers - // against the merged tree afterwards) — dep packages first, current last. - const paths = await collectMetadataPaths(repoRoot); + // Both arms are `resolveCollection`'s answer — one already computed by the + // caller, one computed here. There is no third way to find metadata, and + // that is the whole of this line's design: the previous no-`files` arm + // scanned `/` directly, so a caller that copied the + // routed shape but forgot `files` silently loaded from a directory the + // project's config may never have mentioned. + const paths = options?.files !== undefined + ? [...options.files] + : [...(await resolveCollection(repoRoot)).files]; const loader = new MetaDataLoader({ registry, @@ -117,73 +129,3 @@ export async function loadMemory( return result.root; } - -// Dep packages' metaobjects/ files first (topological order), then current. -async function collectMetadataPaths(repoRoot: string): Promise { - const currentMetaDir = join(repoRoot, ".meta"); - const ws = await discoverWorkspace(repoRoot); - - // Workspace path: walk extends, load dep metaobjects/ dirs first - if (ws !== undefined) { - const currentPkg = ws.packages.find((p) => p.metaDir === currentMetaDir); - if (currentPkg !== undefined && currentPkg.manifest.extends.length > 0) { - const ordered = resolveExtendsOrder(ws, currentMetaDir); - const paths: string[] = []; - for (const pkg of ordered) { - // Each workspace package's metadata lives alongside its .meta/ dir - const pkgRoot = join(pkg.metaDir, ".."); - paths.push(...(await listMetadataFiles(join(pkgRoot, DEFAULT_METADATA_DIR)))); - } - return paths; - } - } - - // Single-package path: scan metaobjects/ at the project root - return listMetadataFiles(join(repoRoot, DEFAULT_METADATA_DIR)); -} - -/** - * Recursively list metadata files (*.json, *.yaml, *.yml) under a directory, - * excluding _pending/ at any level. Subdirectories (e.g. projections/) are - * walked depth-first. Files within a directory are sorted alphabetically for - * deterministic load order; subdirectories are visited after files at the - * same level. - * - * Format selection (parsing) happens downstream in `FileSource` from - * `@metaobjectsdev/metadata`, which infers the parser from file extension. - */ -async function listMetadataFiles(dir: string): Promise { - let entries: string[]; - try { - entries = await readdir(dir); - } catch (err) { - throw new Error(`loadMemory: cannot read ${dir}: ${(err as Error).message}`); - } - const paths: string[] = []; - const subdirs: string[] = []; - // #188: sort the raw `readdir` entries so file order is deterministic across - // runtimes/filesystems (Node vs Bun return different `readdir` orders), matching - // this function's docstring and the metadata package's own `DirectorySource`. - // (Resolution is now order-INDEPENDENT — super-resolve.ts #188 — so this is the - // deterministic-enumeration FLOOR, not the fix; it keeps every derived artifact - // that preserves declaration order, e.g. serialization, stable across runtimes.) - for (const entry of [...entries].sort()) { - if (entry === "_pending") continue; - const full = join(dir, entry); - const s = await stat(full); - if (s.isDirectory()) { - subdirs.push(full); - } else if (s.isFile() && isMetadataFile(entry)) { - paths.push(full); - } - } - // Recurse into subdirectories after collecting files at this level - for (const sub of subdirs.sort()) { - paths.push(...(await listMetadataFiles(sub))); - } - return paths; -} - -function isMetadataFile(name: string): boolean { - return name.endsWith(".json") || name.endsWith(".yaml") || name.endsWith(".yml"); -} diff --git a/server/typescript/packages/sdk/src/metadata-files.ts b/server/typescript/packages/sdk/src/metadata-files.ts new file mode 100644 index 000000000..bcddc4679 --- /dev/null +++ b/server/typescript/packages/sdk/src/metadata-files.ts @@ -0,0 +1,126 @@ +// server/typescript/packages/sdk/src/metadata-files.ts +// +// The project's default directory names, what counts as a metadata file, and +// the one walk that turns a directory into an ordered file list. +// +// **This module imports nothing from its siblings, and that is the point.** +// `resolveCollection` (`collection.ts`) is the single authority on where +// metadata lives, so `memory.ts`'s `loadMemory` must call it — while +// `collection.ts` and `sources.ts` need the constants and the walk below. +// Homing those in `memory.ts` closes an ESM cycle whose failure mode is not a +// warning but a crash: `DEFAULT_SOURCES` (`sources.ts`) reads +// `DEFAULT_METADATA_DIR` at module top level, so the cycle surfaces as +// `ReferenceError: Cannot access 'DEFAULT_METADATA_DIR' before initialization` +// on whichever module the entry point happens to reach first. A leaf both +// sides import is the fix; a lazy `await import()` inside `loadMemory` is not +// — that hides the cycle rather than removing it. +import { extname, join } from "node:path"; +import { readdir, stat } from "node:fs/promises"; + +/** + * The DEFAULT value of `sources` — the directory scanned when + * `.metaobjects/config.json` declares no sources. Scaffold via `meta init`; + * the directory is committed to git. + * + * **A default, and nothing else.** No read path may assume a directory of this + * name exists or is where metadata lives: that question is answered by + * `resolveCollection`, which applies this constant exactly once (via + * `DEFAULT_SOURCES` in `sources.ts`) when a project declares nothing. A + * project that declares `sources` may put its metadata anywhere, and every + * command follows the config. `test/no-hardcoded-metadata-dir.test.ts` is the + * enforcer. + */ +export const DEFAULT_METADATA_DIR = "metaobjects"; + +/** + * Default directory name (relative to project root) for MetaObjects' own + * runtime state: config.json, .gen-state/, package.meta.json, agent docs. + * Scaffold via `meta init`; most contents are committed to git. + * + * Unlike {@link DEFAULT_METADATA_DIR} this one IS a fixed convention — it is + * where the config that answers "where is the metadata?" lives, so it cannot + * itself be configured. + */ +export const DEFAULT_METAOBJECTS_DIR = ".metaobjects"; + +/** Recognized metadata file extensions, matched case-insensitively — mirrors + * `DirectorySource` in `@metaobjectsdev/metadata`, which checks + * `extname().toLowerCase()`. The single definition every metadata-file + * walker in this package uses — and since `resolveSources` (`sources.ts`) + * calls {@link listMetadataFiles} outright rather than keeping a second + * recursive walk of its own, there is exactly one walker to keep honest. */ +export const METADATA_EXTENSIONS = new Set([".json", ".yaml", ".yml"]); + +export function isMetadataFile(name: string): boolean { + return METADATA_EXTENSIONS.has(extname(name).toLowerCase()); +} + +/** Directory excluded at every level of {@link listMetadataFiles} — drafts + * that are deliberately not part of the loaded model. */ +const PENDING_DIR = "_pending"; + +/** + * Recursively list metadata files (*.json, *.yaml, *.yml, matched + * case-insensitively — see `isMetadataFile` above) under a directory, + * excluding _pending/ at any level. Subdirectories (e.g. projections/) are + * walked depth-first. Files within a directory are sorted alphabetically for + * deterministic load order; subdirectories are visited AFTER the files at the + * same level. + * + * That per-level rule is a contract, not an implementation detail. This is the + * order production has always handed the loader, and declaration order survives + * into generated output: `codegen-ts`'s barrel emits from `root.objects()` + * order, and so do the shared `enums.ts`, `meta docs` page ordering and `meta + * export`'s `canonicalSerialize` sibling order. A flat lexicographic sort of + * absolute paths is NOT the same list — it disagrees whenever a subdirectory + * name sorts before a sibling file (`common/` before `meta.users.json`) — so + * `resolveSources` calls this function rather than re-walking and re-sorting. + * Pinned by `test/source-order.test.ts`. + * + * Exported for that gate and for `sources.ts`; not re-exported from the package + * index — `resolveCollection` is the public door. + * + * An entry whose `stat` fails (a dangling symlink, a TOCTOU removal between + * `readdir` and `stat`, an EACCES entry) is SKIPPED, matching `DirectorySource` + * in `@metaobjectsdev/metadata`, which this walk otherwise mirrors. A failure to + * read the directory itself still throws — that is the "you have no metadata + * here" case callers report. + * + * Format selection (parsing) happens downstream in `FileSource` from + * `@metaobjectsdev/metadata`, which infers the parser from file extension. + */ +export async function listMetadataFiles(dir: string): Promise { + let entries: string[]; + try { + entries = await readdir(dir); + } catch (err) { + throw new Error(`cannot read metadata directory ${dir}: ${(err as Error).message}`); + } + const paths: string[] = []; + const subdirs: string[] = []; + // #188: sort the raw `readdir` entries so file order is deterministic across + // runtimes/filesystems (Node vs Bun return different `readdir` orders), matching + // this function's docstring and the metadata package's own `DirectorySource`. + // (Resolution is now order-INDEPENDENT — super-resolve.ts #188 — so this is the + // deterministic-enumeration FLOOR, not the fix; it keeps every derived artifact + // that preserves declaration order, e.g. serialization, stable across runtimes.) + for (const entry of [...entries].sort()) { + if (entry === PENDING_DIR) continue; + const full = join(dir, entry); + // `stat` (not `lstat`/`Dirent.isDirectory()`) so a symlinked subdirectory is + // traversed — `DirectorySource` has always followed symlinks this way. + const s = await stat(full).catch(() => undefined); + if (s === undefined) continue; + if (s.isDirectory()) { + subdirs.push(full); + } else if (s.isFile() && isMetadataFile(entry)) { + paths.push(full); + } + } + // Recurse into subdirectories after collecting files at this level. + // `subdirs` is already in sorted order (built from the sorted `entries` above). + for (const sub of subdirs) { + paths.push(...(await listMetadataFiles(sub))); + } + return paths; +} diff --git a/server/typescript/packages/sdk/src/scope.ts b/server/typescript/packages/sdk/src/scope.ts new file mode 100644 index 000000000..26d7da398 --- /dev/null +++ b/server/typescript/packages/sdk/src/scope.ts @@ -0,0 +1,95 @@ +// server/typescript/packages/sdk/src/scope.ts +// +// Phase-1 metadata-source-resolution — the scope pattern engine. +// +// A pure, no-I/O module deciding whether a fully-qualified node name falls +// inside a consumer's declared `include`/`exclude` scope. Source resolution +// and discovery (later phase-1 tasks) build on this; a cross-language +// conformance corpus pins its semantics, so exact pattern behavior matters. + +// NOTE: PACKAGE_SEPARATOR is NOT re-exported from the browser-safe +// `@metaobjectsdev/metadata/constants` barrel (that barrel only re-exports +// the per-concern `*-constants.ts` modules; `PACKAGE_SEPARATOR` lives in +// `shared/structural.ts`, exported from the package root). This package +// (`@metaobjectsdev/sdk`) is server-side, not a `client/web/**` browser +// package, so importing metamodel values from the root — the same thing +// `memory.ts` and `forge-types.ts` in this package already do — is correct. +import { PACKAGE_SEPARATOR, ParseError, codeSource } from "@metaobjectsdev/metadata"; + +/** A consumer-side output filter over fully-qualified node names. */ +export interface Scope { + /** Absent or empty means "everything". */ + readonly include?: readonly string[]; + /** Applied after `include`. */ + readonly exclude?: readonly string[]; +} + +export interface CompiledScope { + readonly include: readonly RegExp[]; + readonly exclude: readonly RegExp[]; +} + +/** One package segment: any run of characters containing no separator char. */ +const SEGMENT = "[^:]+"; +/** One or more segments, separator-joined — the `**` expansion. */ +const SEGMENTS = `${SEGMENT}(?:${PACKAGE_SEPARATOR}${SEGMENT})*`; + +function escapeLiteral(text: string): string { + return text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** Compile one segment. `**` spans segments; `*` never crosses a separator. */ +function compileSegment(segment: string, pattern: string): string { + if (segment.length === 0) { + throw new ParseError(`empty segment in scope pattern "${pattern}"`, { + code: "ERR_SCOPE_PATTERN_INVALID", + source: codeSource("compileSegment"), + }); + } + // A segment surviving the split on the two-character PACKAGE_SEPARATOR + // ("::") can still contain a lone ":" when the pattern has an odd colon + // run — e.g. "acme:::Order".split("::") => ["acme", ":Order"]. SEGMENT + // ([^:]+) already excludes ":" from a well-formed segment, so a leftover + // ":" here means the separator was malformed, not that ":" is meant + // literally. Left unchecked, escapeLiteral treats it as a literal + // character and compiles a regex requiring three colons in a row — which + // no legal "::"-joined fully-qualified name can ever contain, so the + // pattern silently matches nothing instead of failing loud. + if (segment.includes(":")) { + throw new ParseError( + `scope pattern "${pattern}" has a malformed separator (an odd run of ":") — segments are joined by "::", never a single ":"`, + { code: "ERR_SCOPE_PATTERN_INVALID", source: codeSource("compileSegment") }, + ); + } + if (segment === "**") return `(?:${SEGMENTS})`; + // `*` inside a segment matches any characters except the separator char. + return segment.split("*").map(escapeLiteral).join("[^:]*"); +} + +export function compilePattern(pattern: string): RegExp { + if (pattern.length === 0) { + throw new ParseError(`scope pattern must not be empty`, { + code: "ERR_SCOPE_PATTERN_INVALID", + source: codeSource("compilePattern"), + }); + } + const body = pattern + .split(PACKAGE_SEPARATOR) + .map((segment) => compileSegment(segment, pattern)) + .join(PACKAGE_SEPARATOR); + return new RegExp(`^${body}$`); +} + +export function compileScope(scope: Scope): CompiledScope { + return { + include: (scope.include ?? []).map(compilePattern), + exclude: (scope.exclude ?? []).map(compilePattern), + }; +} + +/** True when `fqn` is inside the scope. An empty `include` means everything. */ +export function matchesScope(fqn: string, compiled: CompiledScope): boolean { + const included = compiled.include.length === 0 || compiled.include.some((re) => re.test(fqn)); + if (!included) return false; + return !compiled.exclude.some((re) => re.test(fqn)); +} diff --git a/server/typescript/packages/sdk/src/sources.ts b/server/typescript/packages/sdk/src/sources.ts new file mode 100644 index 000000000..5f301d85a --- /dev/null +++ b/server/typescript/packages/sdk/src/sources.ts @@ -0,0 +1,160 @@ +// server/typescript/packages/sdk/src/sources.ts +// +// Phase-1 metadata-source-resolution — source spec resolution. +// +// Turns a declared source SET (`.metaobjects/config.json`'s `sources`) into a +// canonically-ordered, de-duplicated list of metadata file paths. The FULL +// result — including which spec each entry attributes to — is a pure +// function of the source SET, never of declaration order: permuting `specs` +// cannot change the output, even when two specs overlap on the same file. +// `test/order-independence.test.ts` pins that (the design's linchpin), so the +// canonical spec ordering below is load-bearing. +// +// Canonical is NOT the same as "flat-sorted". Within one directory spec the +// order is `listMetadataFiles`'s (metadata-files.ts) — files at a level, then +// that level's subdirectories, depth-first — because that is the order production +// has always handed the loader, and declaration order survives into generated +// output (the barrel's export list, the shared `enums.ts`, `meta docs` page +// order, `meta export`'s sibling order). A flat sort of absolute paths +// silently reorders any project with a subdirectory whose name sorts before a +// sibling file. Across specs, order is decided by spec CONTENT, which is what +// keeps the whole result permutation-invariant. `test/source-order.test.ts` +// pins both halves. +import { stat } from "node:fs/promises"; +import { isAbsolute, resolve } from "node:path"; +import { ParseError, codeSource } from "@metaobjectsdev/metadata"; +import { DEFAULT_METADATA_DIR, listMetadataFiles } from "./metadata-files.js"; + +/** Tagged union of source kinds. `resource` and `package` are declared now so + * the config shape is stable across phases; only `path` resolves in phase 1 — + * `resource`/`package` throw `ERR_SOURCE_KIND_UNSUPPORTED`. */ +export type SourceSpec = + | { readonly path: string } + | { readonly resource: string } + | { readonly package: string }; + +export interface ResolvedSource { + /** Absolute path of one metadata file. */ + readonly file: string; + /** The spec that contributed it — provenance for diagnostics. */ + readonly spec: SourceSpec; +} + +/** Used when `sources` is absent or empty in `.metaobjects/config.json`. A + * DEFAULT, never a requirement — a project that declares `sources` explicitly + * need not include the default directory at all. Built from + * `DEFAULT_METADATA_DIR` (`metadata-files.ts`'s single definition) rather than + * restating that name here: a second independent encoding of the same + * default would let `resolveCollection`'s "does the default dir exist" + * check (`collection.ts`) desync from what `resolveSources` actually + * resolves the moment the default ever changed — silently reproducing the + * "two code paths disagree about where metadata lives" class of bug this + * whole mechanism exists to eliminate. */ +export const DEFAULT_SOURCES: readonly SourceSpec[] = [{ path: DEFAULT_METADATA_DIR }]; + +/** Narrows `spec` to its `path` arm, throwing `ERR_SOURCE_KIND_UNSUPPORTED` + * for `resource`/`package` — phase 1 resolves `path` only. Returns rather than + * asserting so one call both validates and narrows: an `asserts` signature has + * to be re-invoked wherever TypeScript's control-flow analysis cannot carry the + * narrowing, which is a language workaround masquerading as a second check. */ +function toPathSpec(spec: SourceSpec): { readonly path: string } { + if ("path" in spec) return spec; + const kind = "resource" in spec ? "resource" : "package"; + throw new ParseError( + `source kind "${kind}" is not supported by this toolchain yet; use a "path" source`, + { code: "ERR_SOURCE_KIND_UNSUPPORTED", source: codeSource("resolveSources") }, + ); +} + +/** + * The declared source SET in CANONICAL order — kind-validated, then sorted by + * spec CONTENT rather than by declaration order. + * + * This is the ONE place declaration order is discarded, and the module's "pure + * function of the SET" invariant rests on it: the emitted file order, the spec + * attributed to a file two specs both reach, and which of several unresolvable + * paths reports its `ERR_SOURCE_UNRESOLVED` first are all decided here. + * Validation runs across the WHOLE list before any sorting or filesystem I/O — + * interleaved with resolution, which error code came back would depend on + * declaration order, contradicting that same invariant. + * + * Exported because `resolveCollection` derives `sourceRoots` from the declared + * specs and must use this identical ordering; a second sort would be a second + * definition of "canonical". + */ +export function orderedPathSpecs(specs: readonly SourceSpec[]): { readonly path: string }[] { + return specs.map(toPathSpec).sort((a, b) => { + const [ja, jb] = [JSON.stringify(a), JSON.stringify(b)]; + return ja < jb ? -1 : ja > jb ? 1 : 0; + }); +} + +/** + * Where a declared `path` source lives on disk: absolute as written, otherwise + * relative to the DECLARING config's directory — never to ambient + * `process.cwd()`. + * + * One definition, because this expression *is* the rule for where a declared + * source lives, which is the single piece of knowledge this module exists to + * own. A caller that needs a source's root directory (rather than its files) + * calls this rather than restating it. + */ +export function resolveSpecPath(configDir: string, spec: { readonly path: string }): string { + return isAbsolute(spec.path) ? spec.path : resolve(configDir, spec.path); +} + +/** + * Resolve a declared source SET to a canonically-ordered list of metadata files. + * + * The full result — each entry's `.file` AND its `.spec` — is a pure function + * of the SET of `specs`: permuting `specs` cannot change the output. One thing + * makes that hold: the specs are processed in CONTENT order + * (`JSON.stringify(spec)`, ascending) rather than declared order, so both the + * emitted file order and the spec attributed to a file overlapping two specs + * are decided by content alone. Declared order carries no information anywhere + * in this function. + * + * Within one directory spec the file order is `listMetadataFiles`'s — files at + * a level, then that level's subdirectories, depth-first. That is deliberately + * NOT a flat sort of absolute paths: see the file header, and + * `test/source-order.test.ts`. + * + * Only `path` specs resolve in phase 1: a directory is walked recursively, a + * file is taken as-is. An unresolvable `path` throws `ERR_SOURCE_UNRESOLVED` + * rather than silently contributing nothing; `resource`/`package` specs throw + * `ERR_SOURCE_KIND_UNSUPPORTED`. + * + * @param configDir absolute directory of the declaring config (the parent of + * `.metaobjects/`) — relative `path` specs resolve against it, never against + * ambient `process.cwd()`. + */ +export async function resolveSources( + configDir: string, + specs: readonly SourceSpec[], +): Promise { + // Kind-validated and content-ordered in one pass — see `orderedPathSpecs`. + const ordered = orderedPathSpecs(specs); + + // Insertion order IS output order — a Map preserves it, so the per-spec walk + // order above survives to the caller. First contributor wins a shared file, + // which is content-determined because `ordered` is. + const byFile = new Map(); + + for (const spec of ordered) { + const target = resolveSpecPath(configDir, spec); + const stats = await stat(target).catch(() => undefined); + if (stats === undefined) { + throw new ParseError( + `source path "${spec.path}" does not exist (resolved to ${target}, relative to ${configDir})`, + { code: "ERR_SOURCE_UNRESOLVED", source: codeSource("resolveSources") }, + ); + } + + const found = stats.isDirectory() ? await listMetadataFiles(target) : [target]; + for (const file of found) { + if (!byFile.has(file)) byFile.set(file, spec); + } + } + + return [...byFile].map(([file, spec]) => ({ file, spec })); +} diff --git a/server/typescript/packages/sdk/test/collection.test.ts b/server/typescript/packages/sdk/test/collection.test.ts new file mode 100644 index 000000000..b60f71ebf --- /dev/null +++ b/server/typescript/packages/sdk/test/collection.test.ts @@ -0,0 +1,134 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveCollection } from "../src/collection.js"; +import { rejectedCode } from "./support/error-code.js"; + +let root: string; +const write = (rel: string, body: string) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), body, "utf8"); +}; +const config = (dir: string, cfg: object) => + write(join(dir, ".metaobjects/config.json"), JSON.stringify({ schema_version: 1, ...cfg })); + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-collection-")); mkdirSync(join(root, ".git")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("resolveCollection", () => { + test("BACK-COMPAT: no sources declared falls back to metaobjects/", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + const c = await resolveCollection(root); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["metaobjects/meta.a.json"]); + }); + + test("BACK-COMPAT: no config at all still finds metaobjects/ in the start dir", async () => { + write("metaobjects/meta.a.json", "{}"); + const c = await resolveCollection(root); + expect(c.files).toHaveLength(1); + }); + + test("a consumer reaches a tree elsewhere in the repo", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui")); + expect(c.configDir).toBe(join(root, "apps/ui")); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); + }); + + test("scope compiles into a predicate the caller passes straight through", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }], scope: { include: ["acme::**"] } }); + const c = await resolveCollection(join(root, "apps/ui")); + expect(c.inScope("acme::Order")).toBe(true); + expect(c.inScope("other::Order")).toBe(false); + }); + + test("an undeclared scope admits everything — the predicate is always defined", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + expect((await resolveCollection(root)).inScope("anything::at::All")).toBe(true); + }); + + test("migrateScope is undefined when not declared", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", {}); + expect((await resolveCollection(root)).inMigrateScope).toBeUndefined(); + }); + + test("migrateScope compiles when declared", async () => { + write("metaobjects/meta.a.json", "{}"); + config(".", { migrate: { scope: ["acme::platform::**"] } }); + const c = await resolveCollection(root); + expect(c.inMigrateScope!("acme::platform::Job")).toBe(true); + // `**` spans any number of segments — `matchesScope` decides, here as + // everywhere: there is exactly one implementation of the pattern grammar. + expect(c.inMigrateScope!("acme::platform::billing::Invoice")).toBe(true); + expect(c.inMigrateScope!("arena::Match")).toBe(false); + }); + + test("an explicit dir overrides discovery", async () => { + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + config("apps/api", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui"), { explicitDir: join(root, "apps/api") }); + expect(c.configDir).toBe(join(root, "apps/api")); + }); + + test("nothing discoverable and no default dir is ERR_COLLECTION_NOT_FOUND", async () => { + mkdirSync(join(root, "apps/ui"), { recursive: true }); + expect(await rejectedCode(resolveCollection(join(root, "apps/ui")))).toBe( + "ERR_COLLECTION_NOT_FOUND", + ); + }); + + test("a malformed config.json rejects rather than silently falling back to metaobjects/", async () => { + write("metaobjects/meta.a.json", "{}"); + // Truncated JSON — config.json EXISTS but cannot be parsed. Must surface + // as a real load failure, never a silent DEFAULT_SOURCES fallback: a + // typo'd config that quietly generates from a possibly-stale + // `metaobjects/` with no diagnostic is worse than the status quo this + // design set out to fix. + write(".metaobjects/config.json", '{ "schema_version": 1, '); + await expect(resolveCollection(root)).rejects.toThrow(SyntaxError); + }); + + test("a .metaobjects/ directory with no config.json still falls back to metaobjects/", async () => { + write("metaobjects/meta.a.json", "{}"); + mkdirSync(join(root, ".metaobjects"), { recursive: true }); // dir exists, file does not + const c = await resolveCollection(root); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["metaobjects/meta.a.json"]); + }); + + test("a bare metaobjects/ is NOT a project boundary — the ancestor config governs", async () => { + // A project boundary is a `.metaobjects/config.json`, nothing else. A + // subdirectory holding only a `metaobjects/` directory declares no project, + // so the nearest ancestor config governs it — including its `sources`, which + // may point nowhere near either directory. The alternative (treating a bare + // directory as a second stop marker) puts a second definition of "where + // metadata lives" back into the walk, which is exactly what + // `resolveCollection` exists to be the only one of. A subdirectory that + // should own its metadata declares a config; `meta init` writes one. + config(".", {}); + write("metaobjects/meta.root.json", "{}"); + write("apps/ui/metaobjects/meta.ui.json", "{}"); + const c = await resolveCollection(join(root, "apps/ui")); + expect(c.configDir).toBe(root); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual([ + "metaobjects/meta.root.json", + ]); + }); + + test("a nearer config wins over an ancestor's", async () => { + // Nearest-ancestor, first-match-wins: a config beside the start dir governs, + // and a `metaobjects/` sitting in an ancestor changes nothing. + write("metaobjects/meta.root.json", "{}"); + write("model/meta.a.json", "{}"); + config("apps/ui", { sources: [{ path: "../../model" }] }); + const c = await resolveCollection(join(root, "apps/ui/src")); + expect(c.configDir).toBe(join(root, "apps/ui")); + expect(c.files.map((f) => f.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); + }); +}); diff --git a/server/typescript/packages/sdk/test/config.test.ts b/server/typescript/packages/sdk/test/config.test.ts index 804e3ed3d..1a17d27d5 100644 --- a/server/typescript/packages/sdk/test/config.test.ts +++ b/server/typescript/packages/sdk/test/config.test.ts @@ -23,14 +23,14 @@ describe("ConfigSchema", () => { test("accepts a path source", () => { const parsed = ConfigSchema.parse({ schema_version: 1, - sources: [{ kind: "path", path: "../shared/.meta" }], + sources: [{ path: "../shared/.meta" }], }); expect(parsed.sources).toHaveLength(1); }); test("accepts a package source", () => { const parsed = ConfigSchema.parse({ schema_version: 1, - sources: [{ kind: "package", package: "@acme/entities" }], + sources: [{ package: "@acme/entities" }], }); expect(parsed.sources).toHaveLength(1); }); @@ -100,3 +100,73 @@ describe("ConfigSchema — migrate block", () => { expect(parsed.migrate?.databaseUrl).toBe("postgres://localhost/db"); }); }); + +describe("ConfigSchema — phase-1 source resolution", () => { + test("accepts a path source", () => { + const p = ConfigSchema.parse({ schema_version: 1, sources: [{ path: "../model" }] }); + expect(p.sources).toEqual([{ path: "../model" }]); + }); + test("accepts resource and package source kinds", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + sources: [{ resource: "acme/model" }, { package: "@acme/model" }], + }); + expect(p.sources).toHaveLength(2); + }); + test("rejects an unknown source kind", () => { + expect(() => ConfigSchema.parse({ schema_version: 1, sources: [{ nope: "x" }] })).toThrow(); + }); + test("rejects a source with an unrecognized extra key (fail-closed, not stripped)", () => { + // A typo'd sibling key must not silently vanish and leave a + // valid-looking single-key source behind — .strict() on every + // SourceSpecSchema arm means an unknown key is a hard parse error, + // matching this project's fail-closed posture elsewhere (ADR-0023). + expect(() => + ConfigSchema.parse({ schema_version: 1, sources: [{ path: "model", pathh: "typo" }] }), + ).toThrow(); + }); + test("accepts a scope block", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + scope: { include: ["acme::**"], exclude: ["acme::internal::**"] }, + }); + expect(p.scope?.include).toEqual(["acme::**"]); + }); + test("scope defaults to undefined (match everything)", () => { + expect(ConfigSchema.parse({ schema_version: 1 }).scope).toBeUndefined(); + }); + test("accepts migrate.scope", () => { + const p = ConfigSchema.parse({ + schema_version: 1, + migrate: { scope: ["acme::platform::**"] }, + }); + expect(p.migrate?.scope).toEqual(["acme::platform::**"]); + }); + test("rejects a typo'd key inside `migrate` — a stripped `scopee` silently means UNSCOPED", () => { + // The nested blocks were `.partial()` under zod's default strip policy, so + // `{ migrate: { scopee: [...], dialect: "postgres" } }` parsed to + // `{ dialect: "postgres" }` — the typo vanished and the run governed the whole + // database. That is the `migrate.scope`-matched-nothing hazard through a + // second door, and the exact fail-open `.strict()` exists to prevent. + expect(() => + ConfigSchema.parse({ + schema_version: 1, + migrate: { scopee: ["acme::platform::**"], dialect: "postgres" }, + }), + ).toThrow(); + }); + test("rejects a typo'd key inside `migrate.d1`", () => { + expect(() => + ConfigSchema.parse({ schema_version: 1, migrate: { d1: { bindingg: "DB" } } }), + ).toThrow(); + }); + test("an existing config with no new keys still parses (back-compat)", () => { + const p = ConfigSchema.parse({ + schema_version: 1, pending_in_git: true, + confidence_thresholds: { pending_promote: 0.8, drift_warn: 0.7 }, + sources: [], extract: {}, + }); + expect(p.sources).toEqual([]); + expect(p.scope).toBeUndefined(); + }); +}); diff --git a/server/typescript/packages/sdk/test/discovery.test.ts b/server/typescript/packages/sdk/test/discovery.test.ts new file mode 100644 index 000000000..b6f205d92 --- /dev/null +++ b/server/typescript/packages/sdk/test/discovery.test.ts @@ -0,0 +1,89 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { discoverCollectionRoot, resolveConfigDir } from "../src/discovery.js"; + +let root: string; +const mk = (rel: string) => mkdirSync(join(root, rel), { recursive: true }); +const cfg = (rel: string) => { + mk(join(rel, ".metaobjects")); + writeFileSync(join(root, rel, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); +}; +/** A `metaobjects/` directory with no config — used below to prove it is + * NOT a stop marker; the walk stops on `.metaobjects/config.json` and the + * `.git` boundary only. */ +const meta = (rel: string) => mk(join(rel, "metaobjects")); + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-discovery-")); mk(".git"); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("discoverCollectionRoot — config marker", () => { + test("finds a config in the start directory", async () => { + cfg("apps/ui"); mk("apps/ui/src"); + expect(await discoverCollectionRoot(join(root, "apps/ui"))).toEqual({ + dir: join(root, "apps/ui"), hasConfig: true, + }); + }); + test("walks up to the nearest ancestor config", async () => { + cfg("apps/ui"); mk("apps/ui/src/deep"); + expect(await resolveConfigDir(join(root, "apps/ui/src/deep"))).toBe(join(root, "apps/ui")); + }); + test("nearest wins over a further ancestor", async () => { + cfg("."); cfg("apps/ui"); mk("apps/ui/src"); + expect(await resolveConfigDir(join(root, "apps/ui/src"))).toBe(join(root, "apps/ui")); + }); + test("stops at the repository boundary — never adopts a parent checkout's config", async () => { + // A config ABOVE the .git boundary must not be found. + const outer = mkdtempSync(join(tmpdir(), "metaobjects-outer-")); + try { + mkdirSync(join(outer, "inner/.git"), { recursive: true }); + mkdirSync(join(outer, ".metaobjects"), { recursive: true }); + writeFileSync(join(outer, ".metaobjects/config.json"), '{"schema_version":1}', "utf8"); + mkdirSync(join(outer, "inner/src"), { recursive: true }); + const start = join(outer, "inner/src"); + expect(await discoverCollectionRoot(start)).toEqual({ dir: start, hasConfig: false }); + } finally { + rmSync(outer, { recursive: true, force: true }); + } + }); + test("a repo-root config IS found from a subdirectory", async () => { + cfg("."); mk("apps/ui"); + expect(await resolveConfigDir(join(root, "apps/ui"))).toBe(root); + }); + test("falls back to the start directory when nothing is found", async () => { + mk("apps/ui"); + const start = join(root, "apps/ui"); + expect(await discoverCollectionRoot(start)).toEqual({ dir: start, hasConfig: false }); + }); +}); + +describe("discoverCollectionRoot — a metadata directory is not a marker", () => { + // `.metaobjects/config.json` is the ONLY stop condition (plus the `.git` + // boundary). A directory that merely holds metadata declares no project, so + // the walk goes straight past it. Anything else would be a second definition + // of "where metadata lives" living outside `resolveCollection`. + test("a LOCAL metaobjects/ does not stop the walk — the ancestor config governs", async () => { + cfg("."); meta("."); meta("apps/ui"); mk("apps/ui/src"); + expect(await discoverCollectionRoot(join(root, "apps/ui"))).toEqual({ + dir: root, hasConfig: true, + }); + }); + test("with no config anywhere, a metaobjects/ up the tree is passed over", async () => { + meta("apps/ui"); mk("apps/ui/src/deep"); + const start = join(root, "apps/ui/src/deep"); + expect(await resolveConfigDir(start)).toBe(start); + }); + test("a config in the SAME directory wins — hasConfig is true", async () => { + cfg("apps/ui"); meta("apps/ui"); + expect(await discoverCollectionRoot(join(root, "apps/ui"))).toEqual({ + dir: join(root, "apps/ui"), hasConfig: true, + }); + }); + test("a nearer config beats a further ancestor's metaobjects/", async () => { + meta("."); cfg("apps/ui"); mk("apps/ui/src"); + expect(await discoverCollectionRoot(join(root, "apps/ui/src"))).toEqual({ + dir: join(root, "apps/ui"), hasConfig: true, + }); + }); +}); diff --git a/server/typescript/packages/sdk/test/dogfood-examples.test.ts b/server/typescript/packages/sdk/test/dogfood-examples.test.ts new file mode 100644 index 000000000..160bc0f0e --- /dev/null +++ b/server/typescript/packages/sdk/test/dogfood-examples.test.ts @@ -0,0 +1,123 @@ +// server/typescript/packages/sdk/test/dogfood-examples.test.ts +// +// Dogfoods `resolveCollection` (reach) + `matchesScope` (scope) against a +// real metadata tree already committed in this repo — zero new metadata +// authored. Three angles, per the task-13 addendum: +// 1. a synthetic consumer elsewhere in a repo reaching the examples tree +// via an absolute-path `sources` entry (the general cross-tree case); +// 2. the examples project's OWN committed config, whose `sources: []` is +// the exact shape every `meta init` scaffold produces — nothing else on +// this branch pins that empty-array-falls-back-to-metaobjects/ path +// against a real committed config; +// 3. scope patterns evaluated over the FQNs the loader actually produced +// for that tree, not string literals invented for the test. +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { basename, join, resolve } from "node:path"; +import { resolveCollection } from "../src/collection.js"; +import { loadMemory } from "../src/memory.js"; +import { DEFAULT_METADATA_DIR, listMetadataFiles } from "../src/metadata-files.js"; +import { compileScope, matchesScope } from "../src/scope.js"; + +// Located relative to this test file (never a hardcoded absolute home path — +// this repo is public) so the test runs unchanged on any checkout. +const EXAMPLES_PROJECT = resolve(import.meta.dir, "../../../../../examples/advanced-modeling"); +// `DEFAULT_METADATA_DIR`, not the literal: this file dogfoods the rule that no +// call site may assume that directory name, so it must not assume it either. +const EXAMPLES = join(EXAMPLES_PROJECT, DEFAULT_METADATA_DIR); + +// The tree holds exactly these three files today (verified at HEAD by the +// controller before dispatching this task) — asserted as a floor ("contains +// all three"), never as an exact `length`, since the example tree is +// documentation and may legitimately grow. +const KNOWN_BASENAMES = ["meta.catalog.yaml", "meta.content.yaml", "meta.prompts.yaml"]; + +/** Shared shape for both dogfood file-set assertions (F22): every resolved + * path sits under `under`, the known files are all present, and the order is + * exactly the walk order the toolchain has always used — asserted by calling + * `listMetadataFiles` on the same tree rather than restating a rule, so a + * consumer reaching this collection from elsewhere gets byte-identical + * generated output to a consumer sitting on top of it. */ +async function assertKnownFileSet(files: readonly string[], under: string): Promise { + expect(files.every((f) => f.startsWith(under))).toBe(true); + const names = files.map((f) => basename(f)); + for (const known of KNOWN_BASENAMES) expect(names).toContain(known); + expect([...files]).toEqual(await listMetadataFiles(under)); +} + +describe("dogfood: a consumer reaches the in-repo examples tree", () => { + let consumer: string; + beforeEach(() => { + consumer = mkdtempSync(join(tmpdir(), "metaobjects-dogfood-")); + mkdirSync(join(consumer, ".git")); + mkdirSync(join(consumer, "apps/ui/.metaobjects"), { recursive: true }); + writeFileSync( + join(consumer, "apps/ui/.metaobjects/config.json"), + JSON.stringify({ schema_version: 1, sources: [{ path: EXAMPLES }] }), + "utf8", + ); + }); + afterEach(() => { + rmSync(consumer, { recursive: true, force: true }); + }); + + test("resolves every metadata file in it", async () => { + const c = await resolveCollection(join(consumer, "apps/ui")); + await assertKnownFileSet(c.files, EXAMPLES); + }); + + test("the resolved set loads without errors", async () => { + const c = await resolveCollection(join(consumer, "apps/ui")); + const root = await loadMemory(c.configDir, { files: c.files }); + expect(root.children().length).toBeGreaterThan(0); + }); +}); + +describe("dogfood: the examples project's own committed config (sources: [])", () => { + test("falls back to metaobjects/ under the project root, exactly like every adopter's default config", async () => { + const c = await resolveCollection(EXAMPLES_PROJECT); + expect(c.configDir).toBe(EXAMPLES_PROJECT); + await assertKnownFileSet(c.files, EXAMPLES); + }); +}); + +describe("dogfood: scope evaluated over real loaded FQNs", () => { + test("acme::learn::** matches every loaded object; acme::* (one segment) matches none; excluding Program* narrows without emptying", async () => { + const c = await resolveCollection(EXAMPLES_PROJECT); + const root = await loadMemory(c.configDir, { files: c.files }); + // Derived from the loaded root, not a hardcoded object-name list — this + // must keep working the moment someone edits the example tree. + const fqns = root.children().map((child) => child.resolutionKey()); + expect(fqns.length).toBeGreaterThan(0); + + const broad = compileScope({ include: ["acme::learn::**"] }); + expect(fqns.every((f) => matchesScope(f, broad))).toBe(true); + + // The discriminating case: `*` never crosses `::`, and every object here + // sits two segments below `acme` — a port that treated `*` as "any + // characters" would pass the assertion above and fail this one. + const tooNarrow = compileScope({ include: ["acme::*"] }); + expect(fqns.every((f) => !matchesScope(f, tooNarrow))).toBe(true); + + const excludingProgram = compileScope({ + include: ["acme::learn::**"], + exclude: ["acme::learn::Program*"], + }); + const actualIncluded = fqns.filter((f) => matchesScope(f, excludingProgram)); + const actualExcluded = fqns.filter((f) => !matchesScope(f, excludingProgram)); + + // Expected sets computed from the same FQN list (never a hardcoded name + // list), by the same rule the pattern encodes: the last `::`-segment + // starts with "Program". + const expectedExcluded = fqns.filter((f) => f.split("::").at(-1)!.startsWith("Program")); + const expectedIncluded = fqns.filter((f) => !f.split("::").at(-1)!.startsWith("Program")); + expect([...actualExcluded].sort()).toEqual([...expectedExcluded].sort()); + expect([...actualIncluded].sort()).toEqual([...expectedIncluded].sort()); + + // Non-vacuous on both sides of the exclude — the example tree does carry + // Program-prefixed and non-Program-prefixed objects today. + expect(expectedExcluded.length).toBeGreaterThan(0); + expect(expectedIncluded.length).toBeGreaterThan(0); + }); +}); diff --git a/server/typescript/packages/sdk/test/memory.test.ts b/server/typescript/packages/sdk/test/memory.test.ts index 435601dd1..2de96af8c 100644 --- a/server/typescript/packages/sdk/test/memory.test.ts +++ b/server/typescript/packages/sdk/test/memory.test.ts @@ -3,6 +3,7 @@ import { mkdtempSync, mkdirSync, writeFileSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { loadMemory } from "../src/memory.js"; +import { rejectedCode } from "./support/error-code.js"; function makeMetaRoot(): string { const root = mkdtempSync(join(tmpdir(), "memory-load-")); @@ -70,6 +71,37 @@ describe("loadMemory", () => { } }); + // C4 — memory.ts's own `isMetadataFile` used to match extensions + // case-SENSITIVELY while sources.ts's (already fixed to mirror + // DirectorySource in @metaobjectsdev/metadata) matched case-insensitively. + // Two metadata-file walkers in one package disagreeing about whether + // `meta.JSON` counts is exactly the drift this package's design exists to + // prevent; memory.ts now imports the shared, case-insensitive + // implementation. This is an intentional BEHAVIOR CHANGE — a file named + // `*.JSON` (previously silently skipped by loadMemory) is now collected. + test("collects a metadata file with an uppercase extension (meta.JSON), case-insensitively", async () => { + const root = makeMetaRoot(); + try { + writeFileSync( + join(root, "metaobjects", "shouty.JSON"), + JSON.stringify({ + metadata: { + package: "test", + children: [ + { object: { name: "Shouty", subType: "entity", children: [] } }, + ], + }, + }), + ); + + const meta = await loadMemory(root); + const shouty = meta.findObject("Shouty"); + expect(shouty).toBeDefined(); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); + test("loads decision children when metadata files contain them", async () => { const root = makeMetaRoot(); try { @@ -100,10 +132,14 @@ describe("loadMemory", () => { } }); - test("throws if metaobjects/ doesn't exist", async () => { + test("nothing to resolve is ERR_COLLECTION_NOT_FOUND, the same code every command reports", async () => { + // `loadMemory` resolves through `resolveCollection` like every other read + // path, so the "you have no metadata here" failure is that function's + // structured code rather than a bare `readdir` ENOENT from a directory + // `loadMemory` picked on its own. const root = mkdtempSync(join(tmpdir(), "memory-load-nodir-")); try { - await expect(loadMemory(root)).rejects.toThrow(/cannot read|ENOENT|no such/i); + expect(await rejectedCode(loadMemory(root))).toBe("ERR_COLLECTION_NOT_FOUND"); } finally { rmSync(root, { recursive: true, force: true }); } @@ -169,8 +205,17 @@ describe("loadMemory", () => { }); }); -describe("loadMemory — cross-package loading via workspace", () => { - test("loads transitive extends: deps from workspace peers", async () => { +// The `package.meta.json` + workspace `extends:` peer walk is RETIRED (design +// §11; `docs/features/metadata-sources.md` → Upgrading). `loadMemory` had two +// ways of finding metadata, one of them implicit and reachable only from a +// particular repository layout. It now has none of its own: it asks +// `resolveCollection`, exactly like every other read path. +// +// These tests pin what the Upgrading section PROMISES about that removal — +// that it fails loudly, and that a declared source replaces it — rather than +// merely deleting the coverage along with the feature. +describe("loadMemory — the retired workspace peer walk", () => { + test("a declared source is the replacement, and it needs no topological order", async () => { const wsRoot = mkdtempSync(join(tmpdir(), "ws-loadmem-")); try { // Workspace setup: shared package + billing package that extends shared @@ -200,16 +245,16 @@ describe("loadMemory — cross-package loading via workspace", () => { }), ); - // billing package: extends shared; defines an Invoice entity - mkdirSync(join(wsRoot, "packages", "billing", ".meta"), { recursive: true }); + // billing package: reaches shared by DECLARING it as a source. The + // shared entry is written SECOND on purpose — `sources` is a set, so + // there is no topological order to reproduce. + mkdirSync(join(wsRoot, "packages", "billing", ".metaobjects"), { recursive: true }); mkdirSync(join(wsRoot, "packages", "billing", "metaobjects"), { recursive: true }); writeFileSync( - join(wsRoot, "packages", "billing", ".meta", "package.meta.json"), + join(wsRoot, "packages", "billing", ".metaobjects", "config.json"), JSON.stringify({ - name: "@acme/billing", - version: "1.0.0", - metaobjectsPackage: "acme::billing", - extends: ["@acme/shared"], + schema_version: 1, + sources: [{ path: "metaobjects" }, { path: "../shared/metaobjects" }], }), ); writeFileSync( @@ -235,8 +280,9 @@ describe("loadMemory — cross-package loading via workspace", () => { } }); - test("single-package mode works unchanged when no workspace present", async () => { - // No workspace config — loadMemory falls back to current package only + test("a project declaring nothing resolves its own default source, and only that", async () => { + // The other side of the removal: with no config and no peer walk, the + // default source is the whole of what loads. const root = makeMetaRoot(); try { writeFileSync( @@ -257,7 +303,12 @@ describe("loadMemory — cross-package loading via workspace", () => { } }); - test("cross-package super: resolves via extends graph", async () => { + test("a model that leaned on the peer walk fails LOUDLY, with ERR_UNRESOLVED_SUPER", async () => { + // The Upgrading section's promise, gated: nothing generates from a + // half-resolved model. `domain` declares no `sources`, so it resolves its + // own default directory and nothing else — and the `extends:` into + // `acme::common` that the workspace walk used to satisfy now names a target + // no loaded file declares. const wsRoot = mkdtempSync(join(tmpdir(), "ws-crossref-")); try { writeFileSync(join(wsRoot, "pnpm-workspace.yaml"), "packages:\n - 'packages/*'\n"); @@ -317,16 +368,59 @@ describe("loadMemory — cross-package loading via workspace", () => { }), ); - const meta = await loadMemory(join(wsRoot, "packages", "domain")); - const widget = meta.ownChildren().find((c) => c.name === "Widget"); - expect(widget).toBeDefined(); - const idField = widget!.ownChildren().find((c) => c.name === "id"); - expect(idField).toBeDefined(); - // super resolved across package boundary - expect(idField!.superResolved).toBeDefined(); - expect(idField!.superResolved!.typeId.subType).toBe("long"); + expect(await rejectedCode(loadMemory(join(wsRoot, "packages", "domain")))).toBe( + "ERR_UNRESOLVED_SUPER", + ); } finally { rmSync(wsRoot, { recursive: true, force: true }); } }); }); + +describe("loadMemory with an explicit file set", () => { + test("loads exactly the supplied files, ignoring any metaobjects/ dir", async () => { + const dir = mkdtempSync(join(tmpdir(), "metaobjects-memory-files-")); + try { + mkdirSync(join(dir, "model"), { recursive: true }); + mkdirSync(join(dir, "metaobjects"), { recursive: true }); + writeFileSync(join(dir, "model/meta.a.json"), JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }] }, + }), "utf8"); + writeFileSync(join(dir, "metaobjects/meta.decoy.json"), JSON.stringify({ + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Decoy", children: [{ "field.string": { name: "id" } }] } }] }, + }), "utf8"); + const root = await loadMemory(dir, { files: [join(dir, "model/meta.a.json")] }); + const names = root.children().map((c) => c.name); + expect(names).toContain("Order"); + expect(names).not.toContain("Decoy"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); + + test("with no `files` option, a project with a metaobjects/ tree still loads exactly as before", async () => { + const root = makeMetaRoot(); + try { + writeFileSync( + join(root, "metaobjects", "domain.json"), + JSON.stringify({ + "metadata.root": { + package: "acme", + children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }, + ], + }, + }), + "utf8", + ); + + const meta = await loadMemory(root); + const names = meta.children().map((c) => c.name); + expect(names).toEqual(["Order"]); + } finally { + rmSync(root, { recursive: true, force: true }); + } + }); +}); diff --git a/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts b/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts new file mode 100644 index 000000000..37f211bcd --- /dev/null +++ b/server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts @@ -0,0 +1,268 @@ +// server/typescript/packages/sdk/test/no-hardcoded-metadata-dir.test.ts +// +// THE enforcer for the rule the whole source-resolution design rests on: +// +// `metaobjects/` is the DEFAULT VALUE of `sources` and nothing else. +// Everywhere else reads the config, through `resolveCollection`. +// +// Eight independent call sites once hardcoded that directory; routing them +// through one authority fixed the eight, and fixed nothing about the ninth +// somebody adds next month. This test is the part that lasts. It walks the +// `sdk` and `cli` source trees and fails when a file outside the allowlist +// names the directory in CODE. +// +// Three properties keep it from becoming a gate that passes because it checks +// nothing: +// +// 1. The allowlist is file + REASON. A fifth entry costs a sentence explaining +// why that file is allowed to know the name. +// 2. A STALE entry fails. An allowlisted file that no longer contains the +// reference silently re-opens the hole it was covering, so the allowlist +// must be exact in both directions. +// 3. Comments are excluded, so the paragraph explaining the rule is not itself +// a violation — and that exclusion is tested against a real file that +// mentions the directory only in a comment, not merely asserted. +// +// WHAT IT DOES NOT CATCH — write these down or the guard becomes a claim rather +// than a check. Measured, not assumed (each row was run): +// +// CAUGHT join(d, "metaobjects") a plain literal, either quote +// CAUGHT `${d}/metaobjects` a template literal, end or mid +// CAUGHT "no metaobjects/ here" a message naming the directory +// MISSED join(d, "meta" + "objects") any computed spelling +// MISSED const N = "meta"; N + "objects" the same, through a variable +// MISSED "author under metaobjects" the word with no trailing `/` +// MISSED /https:\/\//metaobjects/ a regex literal containing `//` +// +// The third is deliberate, not an oversight: `metaobjects` followed by a space +// is the PRODUCT name far more often than a path ("the metaobjects ledger", +// the `metaobjects:` error prefix), and three such lines were the guard's first +// false positives. No lexical rule separates them. The computed-spelling misses +// are the honest ceiling of a source-text check — this catches the way the +// violation is actually written, which is how all eight original ones were +// written, and it will not catch someone evading it on purpose. +// +// The fourth is a real blind spot in `stripComments`, not a deliberate +// tradeoff: a regex literal containing `//` (e.g. `/https:\/\//`) drives the +// stripper into line-comment state, same as a real `//`, and blanks the rest +// of that physical line — so a violation sitting to its right on the same +// line is silently missed. The stripper has no regex-literal-vs-division +// disambiguation (that requires knowing the preceding token, which a +// character-at-a-time scan does not track). No such construct exists in +// either scanned tree today. +// +// It also scans TypeScript SOURCE only: the four other language ports, the +// `docs/` tree, and JSON/YAML fixtures are outside it. +import { describe, test, expect } from "bun:test"; +import { readFileSync } from "node:fs"; +import { readdir } from "node:fs/promises"; +import { join, relative, resolve, sep } from "node:path"; +import { DEFAULT_METADATA_DIR } from "../src/metadata-files.js"; + +/** `packages/`, so both trees are reachable from this one test file. */ +const PACKAGES = resolve(import.meta.dirname, "../.."); +const TREES = [join(PACKAGES, "sdk", "src"), join(PACKAGES, "cli", "src")]; + +/** + * The complete set of files permitted to name the default metadata directory, + * each with the reason it is permitted. Adding an entry is a deliberate act: + * write down why that file needs to know, or route it through + * `resolveCollection` instead. + * + * Paths are relative to `packages/`. + */ +const ALLOWED: ReadonlyMap = new Map([ + [ + "sdk/src/metadata-files.ts", + "the constant's single definition — DEFAULT_METADATA_DIR is declared here and nowhere else", + ], + [ + "sdk/src/sources.ts", + "DEFAULT_SOURCES — THE default, the value config resolution applies when a project declares no sources", + ], + [ + "sdk/src/collection.ts", + "inside resolveCollection, the one authority: it APPLIES that default (the does-it-exist probe and the ERR_COLLECTION_NOT_FOUND text naming it)", + ], + [ + "sdk/src/index.ts", + "package barrel — a bare re-export of the constant, so `meta init` can import it instead of restating the literal. No use.", + ], + [ + "cli/src/commands/init.ts", + "the scaffolder WRITING the default layout — creating that directory, never assuming one exists", + ], + [ + "sdk/src/agent-docs/body.ts", + "the agent-docs PROSE `meta init` scaffolds beside that layout — documentation content, reachable by no read path; it teaches the default a fresh project gets. A project that declares `sources` elsewhere is given docs that name the default, which is a known wording gap, not a resolution one.", + ], +]); + +// --------------------------------------------------------------------------- +// Comment stripping +// --------------------------------------------------------------------------- + +/** + * Blank out `//` and block comments, preserving every newline so reported line + * numbers stay true. + * + * String and template literals are tracked so a `//` inside one is not mistaken + * for a comment (`"https://example.com"` must keep its text). Quote states also + * reset at a newline: an unterminated quote — which the stripper could only + * reach by mis-scanning something exotic — then costs one line rather than the + * rest of the file. + */ +function stripComments(src: string): string { + type State = "code" | "line" | "block" | "sq" | "dq" | "tpl"; + let state: State = "code"; + const out: string[] = []; + for (let i = 0; i < src.length; i++) { + const c = src[i]!; + const next = src[i + 1]; + if (state === "code") { + if (c === "/" && next === "/") { state = "line"; out.push(" ", " "); i++; continue; } + if (c === "/" && next === "*") { state = "block"; out.push(" ", " "); i++; continue; } + if (c === "'") state = "sq"; + else if (c === '"') state = "dq"; + else if (c === "`") state = "tpl"; + out.push(c); + continue; + } + if (state === "line") { + if (c === "\n") { state = "code"; out.push(c); } else out.push(" "); + continue; + } + if (state === "block") { + if (c === "*" && next === "/") { state = "code"; out.push(" ", " "); i++; continue; } + out.push(c === "\n" ? c : " "); + continue; + } + // sq / dq / tpl — literal text is kept verbatim; the opening quote was + // consumed by the `code` branch above, so a matching quote here CLOSES. + out.push(c); + if (c === "\\" && next !== undefined) { out.push(next); i++; continue; } + const closer = state === "sq" ? "'" : state === "dq" ? '"' : "`"; + if (c === closer) { state = "code"; continue; } + if (c === "\n" && state !== "tpl") state = "code"; + } + return out.join(""); +} + +// --------------------------------------------------------------------------- +// Violation detection +// --------------------------------------------------------------------------- + +/** A preceding character meaning the word is part of a LONGER token, or names + * the STATE directory rather than the metadata one: `.metaobjects` (a fixed + * convention with its own constant), `@metaobjectsdev/sdk`. */ +const NOT_A_DIR_BEFORE = /[A-Za-z0-9_.@]/; + +/** A following character meaning this really is a path segment: a separator, or + * the quote that ends the literal (`join(d, "metaobjects")`). + * + * Everything else is the PRODUCT name in prose — "the metaobjects ledger", + * "reach for metaobjects metadata", the `metaobjects:` error prefix — or a + * longer token (`metaobjectsdev`, `metaobjects.config.ts`, + * `metaobjects-authoring`). Requiring this is what keeps the guard from + * convicting the product's own name, and it is also the guard's sharpest + * limit: a message that says "under metaobjects" with no trailing slash is + * indistinguishable, lexically, from prose about the product. */ +const PATH_SEGMENT_AFTER = /[/"'`]/; + +/** Every line of `code` naming the default directory, as a path literal or via + * the constant. `code` must already have its comments stripped. */ +function violationLines(code: string): number[] { + const hits = new Set(); + const lineOf = (index: number): number => code.slice(0, index).split("\n").length; + + for (const m of code.matchAll(/DEFAULT_METADATA_DIR/g)) hits.add(lineOf(m.index)); + + for (const m of code.matchAll(new RegExp(DEFAULT_METADATA_DIR, "g"))) { + const before = m.index === 0 ? "" : code[m.index - 1]!; + const after = code[m.index + DEFAULT_METADATA_DIR.length] ?? ""; + if (NOT_A_DIR_BEFORE.test(before)) continue; + if (!PATH_SEGMENT_AFTER.test(after)) continue; + hits.add(lineOf(m.index)); + } + return [...hits].sort((a, b) => a - b); +} + +async function tsFiles(dir: string): Promise { + const out: string[] = []; + for (const entry of (await readdir(dir, { withFileTypes: true })).sort((a, b) => + a.name < b.name ? -1 : 1, + )) { + const full = join(dir, entry.name); + if (entry.isDirectory()) out.push(...(await tsFiles(full))); + else if (entry.name.endsWith(".ts") || entry.name.endsWith(".tsx")) out.push(full); + } + return out; +} + +/** Every scanned file naming the directory in code, keyed by `packages/`-relative + * path (always `/`-separated, so the allowlist reads the same on any platform). */ +async function scan(): Promise> { + const found = new Map(); + for (const tree of TREES) { + for (const file of await tsFiles(tree)) { + const lines = violationLines(stripComments(readFileSync(file, "utf8"))); + if (lines.length > 0) found.set(relative(PACKAGES, file).split(sep).join("/"), lines); + } + } + return found; +} + +describe("`metaobjects/` is a config default and nothing else", () => { + test("no file outside the allowlist names the default metadata directory", async () => { + const offenders = [...(await scan()).entries()] + .filter(([file]) => !ALLOWED.has(file)) + .map(([file, lines]) => `${file}:${lines.join(",")}`); + expect(offenders).toEqual([]); + }); + + test("every allowlisted file still contains the reference it was allowed for", async () => { + const found = await scan(); + // A stale entry is not cosmetic: it is an allowlisted hole nobody is + // watching, and the next file to take that path inherits the exemption. + expect([...ALLOWED.keys()].filter((f) => !found.has(f))).toEqual([]); + }); + + test("every allowlist entry carries a reason", () => { + expect([...ALLOWED].filter(([, why]) => why.trim().length < 20).map(([f]) => f)).toEqual([]); + }); + + test("a comment-only mention is not a violation — proven against a real file", () => { + // `detect-stack.ts` explains, in a comment, that it reads the resolved + // collection "rather than assuming `metaobjects/`". Saying so is the + // opposite of a violation, and the guard must not convict it. + const file = join(PACKAGES, "cli", "src", "lib", "detect-stack.ts"); + const raw = readFileSync(file, "utf8"); + expect(raw).toContain(`${DEFAULT_METADATA_DIR}/`); // the mention is really there + expect(violationLines(stripComments(raw))).toEqual([]); // and it is in a comment + }); + + test("the stripper removes comments without eating code", () => { + const cases: [string, boolean][] = [ + [`// join(dir, "${DEFAULT_METADATA_DIR}")`, false], + [`/* a ${DEFAULT_METADATA_DIR}/ tree */`, false], + [`/** ${DEFAULT_METADATA_DIR}/ */\nconst a = 1;`, false], + [`const p = join(dir, "${DEFAULT_METADATA_DIR}");`, true], + [`const p = \`\${d}/${DEFAULT_METADATA_DIR}\`;`, true], + [`const msg = "no ${DEFAULT_METADATA_DIR}/ here";`, true], + // A `//` inside a string is not a comment: the literal must survive. + [`const u = "https://x/${DEFAULT_METADATA_DIR}/y";`, true], + // Longer tokens that merely contain the word are never violations. + [`import x from "@${DEFAULT_METADATA_DIR}dev/sdk";`, false], + [`const f = "${DEFAULT_METADATA_DIR}.config.ts";`, false], + [`const d = ".${DEFAULT_METADATA_DIR}/config.json";`, false], + [`const s = ".claude/skills/${DEFAULT_METADATA_DIR}-authoring";`, false], + // The PRODUCT name in prose is not a directory reference. This is the + // guard's deliberate blind spot, pinned so it stays deliberate. + [`log.error("the ${DEFAULT_METADATA_DIR} ledger is absent");`, false], + [`throw new Error("${DEFAULT_METADATA_DIR}: could not resolve");`, false], + ]; + for (const [src, isViolation] of cases) { + expect(violationLines(stripComments(src)).length > 0).toBe(isViolation); + } + }); +}); diff --git a/server/typescript/packages/sdk/test/order-independence.test.ts b/server/typescript/packages/sdk/test/order-independence.test.ts new file mode 100644 index 000000000..9d4b149e0 --- /dev/null +++ b/server/typescript/packages/sdk/test/order-independence.test.ts @@ -0,0 +1,192 @@ +// server/typescript/packages/sdk/test/order-independence.test.ts +// +// The order-independence gate — the linchpin of the whole design. The +// premise everything else rests on: resolution is a pure function of the +// declared source SET, so the order sources are declared in carries no +// information. That is why the design has no ordered-list semantics, no +// topological sort, no cycle detection, and no diamond-dependency problem. +// +// The premise splits into THREE layers, and this file is the design's +// documentation of record on how each one is satisfied — deliberately not +// collapsed into one over-broad assertion, because two earlier drafts of +// this gate got that collapse wrong in opposite directions: +// 1. `resolveSources` CANONICALIZES file order — it walks the specs in +// CONTENT order rather than declared order, so every permutation of a +// declared source SET collapses to the same file list before the loader +// ever runs. Test 1 pins this directly. (What that canonical order IS — +// per-directory-level, files before subdirectories, never a flat sort of +// absolute paths — is a separate contract, pinned by +// `source-order.test.ts`. This file only asserts it does not depend on +// declaration order.) +// 2. The LOADER resolves CONTENT order-independently, given whatever file +// list it's handed — including an overlay arriving before its base. +// `_partitionOverlayLast` is the mechanism (stable-partitions +// overlay-only sources to the end before the parse loop runs); test 2 +// proves it by permuting FILE PATHS directly into `FileSource[]`, +// bypassing `resolveSources` entirely (routing through it would erase +// all order variation before the loader ever saw it, and reach +// overlay-before-base in zero of the six permutations — an earlier +// draft of this test did exactly that and passed vacuously). Test 2 +// compares CONTENT — each top-level object's own serialization, keyed +// by name — not the whole tree, for the reason in point 3. +// 3. SIBLING ORDER of unrelated top-level nodes (e.g. which of two +// unrelated entities appears first in `MetaRoot`'s `children` array) +// follows raw input order and is DELIBERATELY NOT asserted here. It is +// not a design claim: `canonicalSerialize`'s own contract +// (serializer-json.ts:159-167) promises exactly two normalizations — +// alphabetical `@`-attr keys and a trailing newline — and says nothing +// about sibling ordering; `serializeNodeInner` emits `ownChildren()` in +// whatever order the tree holds them. It also doesn't need to be a +// claim: production never hands the loader a permuted list — layer 1 +// sorts first. A prior draft of test 2 asserted whole-tree +// `canonicalSerialize` equality across all six permutations and failed +// on unmodified code for exactly this reason (Order vs Customer swap), +// even though content resolution was correct in every case — that was +// the amended test inventing a bar the design never set, not a real +// defect. +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSources, type SourceSpec } from "../src/sources.js"; + +let root: string; +const write = (rel: string, body: object) => { + mkdirSync(join(root, rel, ".."), { recursive: true }); + writeFileSync(join(root, rel), JSON.stringify(body), "utf8"); +}; + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "metaobjects-order-")); + // A base declaration, an overlay onto it, and an independent third file — + // the shapes whose merge is order-sensitive if anything is. + write("a/meta.base.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", children: [{ "field.string": { name: "id" } }] } }] }, + }); + write("b/meta.overlay.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Order", overlay: true, children: [ + { "field.string": { name: "note" } }] } }] }, + }); + write("c/meta.other.json", { + "metadata.root": { package: "acme", children: [ + { "object.entity": { name: "Customer", children: [{ "field.string": { name: "id" } }] } }] }, + }); +}); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +function permutations(items: T[]): T[][] { + if (items.length <= 1) return [items]; + const out: T[][] = []; + for (let i = 0; i < items.length; i++) { + const rest = [...items.slice(0, i), ...items.slice(i + 1)]; + for (const p of permutations(rest)) out.push([items[i]!, ...p]); + } + return out; +} + +describe("permutations helper", () => { + test("produces 6 distinct orderings of 3 items", () => { + // Sanity-check the helper itself, not just its output length: 6 entries + // that were secretly duplicates would let both gates below pass + // vacuously without ever exercising a real reordering. + const items = ["a", "b", "c"]; + const perms = permutations(items); + expect(perms).toHaveLength(6); + const distinct = new Set(perms.map((p) => p.join(","))); + expect(distinct.size).toBe(6); + }); +}); + +describe("order independence", () => { + test("resolveSources output is identical across every spec permutation", async () => { + const specs: SourceSpec[] = [{ path: "a" }, { path: "b" }, { path: "c" }]; + const perms = permutations(specs); + expect(perms).toHaveLength(6); + + const results = await Promise.all(perms.map((p) => resolveSources(root, p))); + expect(results).toHaveLength(6); + + // Deep-equal on the FULL ResolvedSource[] — .spec included, not just + // .file. T4's de-dup tie-break is already content-based (compares + // JSON.stringify(spec)), so the full structure is order-free too; a + // .file-only assertion would leave this gate narrower than the property + // it exists to prove. + const expected = results[0]!; + for (let i = 1; i < results.length; i++) { + expect( + results[i], + `permutation ${i} (${JSON.stringify(perms[i])}) diverged from permutation 0 (${JSON.stringify(perms[0])})`, + ).toEqual(expected); + } + }); + + test("the loader resolves content order-independently given a permuted file list, including overlay-before-base", async () => { + const { MetaDataLoader, composeRegistry, coreProviders, canonicalSerialize } = + await import("@metaobjectsdev/metadata"); + const { FileSource } = await import("@metaobjectsdev/metadata/core"); + + // Permute the FILE PATHS directly — deliberately bypassing + // resolveSources(), whose own sort would erase all order variation + // before the loader ever saw it (see the file header). Building + // FileSource[] straight from these paths is what actually reaches an + // overlay-before-base ordering. + const basePath = join(root, "a/meta.base.json"); + const overlayPath = join(root, "b/meta.overlay.json"); + const otherPath = join(root, "c/meta.other.json"); + const perms = permutations([basePath, overlayPath, otherPath]); + expect(perms).toHaveLength(6); + + // Confirm the permutation actually reaches the shape this test exists + // to cover: half of the six orderings must place the overlay-only file + // before its base, or this gate would be no stronger than test 1 above. + const overlayBeforeBase = perms.filter( + (p) => p.indexOf(overlayPath) < p.indexOf(basePath), + ).length; + expect(overlayBeforeBase).toBe(3); + + const label = (p: string[]): string => + JSON.stringify(p.map((f) => f.replace(root + "/", ""))); + + // Per permutation: a name-keyed map of each top-level object's OWN + // canonical serialization. Keying by NAME rather than comparing the + // whole root (or relying on array position) makes the comparison + // insensitive to sibling order by construction — see point 3 in the + // file header — while still catching any real content difference, + // which is the property this test exists to prove. + const perObject: Map[] = []; + for (const p of perms) { + const loader = new MetaDataLoader({ registry: composeRegistry(coreProviders) }); + const result = await loader.load(p.map((file) => new FileSource(file))); + expect(result.errors, `permutation ${label(p)} errored`).toHaveLength(0); + + const byName = new Map(); + for (const child of result.root.ownChildren()) { + byName.set(child.name, canonicalSerialize(child)); + } + perObject.push(byName); + + // The overlay's contribution must have actually landed on Order in + // EVERY permutation — this is the assertion that disabling + // `_partitionOverlayLast` breaks (3 of 6 permutations throw + // ERR_OVERLAY_NO_TARGET without it, dropping this field entirely; see + // the break-and-revert evidence in the task report). The empty-errors + // check above already catches the hard-failure case; this confirms + // the MERGE actually happened, not merely that nothing errored. + expect( + byName.get("Order"), + `permutation ${label(p)} — Order is missing the overlay's note field`, + ).toContain('"name": "note"'); + } + expect(perObject).toHaveLength(6); + + const expected = perObject[0]!; + for (let i = 1; i < perObject.length; i++) { + expect( + perObject[i], + `permutation ${i} (${label(perms[i]!)}) resolved different CONTENT than permutation 0 (${label(perms[0]!)})`, + ).toEqual(expected); + } + }); +}); diff --git a/server/typescript/packages/sdk/test/scope-conformance.test.ts b/server/typescript/packages/sdk/test/scope-conformance.test.ts new file mode 100644 index 000000000..c7334a555 --- /dev/null +++ b/server/typescript/packages/sdk/test/scope-conformance.test.ts @@ -0,0 +1,29 @@ +// server/typescript/packages/sdk/test/scope-conformance.test.ts +import { describe, test, expect } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { compileScope, matchesScope, type Scope } from "../src/scope.js"; + +interface Case { + name: string; + scope: Scope; + expect: Array<{ fqn: string; matches: boolean }>; +} + +const CORPUS = join(import.meta.dir, "../../../../../fixtures/scope-conformance/cases.json"); +const cases = (JSON.parse(readFileSync(CORPUS, "utf8")) as { cases: Case[] }).cases; + +describe("scope-conformance corpus", () => { + test("corpus is non-empty (a silent zero-case run is a failed gate)", () => { + expect(cases.length).toBeGreaterThan(0); + }); + for (const c of cases) { + test(c.name, () => { + const compiled = compileScope(c.scope); + for (const e of c.expect) { + expect({ fqn: e.fqn, matches: matchesScope(e.fqn, compiled) }) + .toEqual({ fqn: e.fqn, matches: e.matches }); + } + }); + } +}); diff --git a/server/typescript/packages/sdk/test/scope.test.ts b/server/typescript/packages/sdk/test/scope.test.ts new file mode 100644 index 000000000..50967ed4c --- /dev/null +++ b/server/typescript/packages/sdk/test/scope.test.ts @@ -0,0 +1,103 @@ +import { describe, test, expect } from "bun:test"; +import { compileScope, matchesScope, type Scope } from "../src/scope.js"; +import { errorCode } from "./support/error-code.js"; + +const match = (fqn: string, scope: Scope) => matchesScope(fqn, compileScope(scope)); + +describe("compileScope / matchesScope", () => { + test("empty include matches everything", () => { + expect(match("acme::commerce::Order", {})).toBe(true); + }); + + test("* matches exactly one segment", () => { + const s: Scope = { include: ["acme::*"] }; + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::commerce::Order", s)).toBe(false); + }); + + test("** matches one or more segments", () => { + const s: Scope = { include: ["acme::**"] }; + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme", s)).toBe(false); + expect(match("other::Order", s)).toBe(false); + }); + + test("* within a segment matches a partial name but never crosses ::", () => { + const s: Scope = { include: ["acme::Order*"] }; + expect(match("acme::OrderLine", s)).toBe(true); + expect(match("acme::Order", s)).toBe(true); + expect(match("acme::deep::OrderLine", s)).toBe(false); + }); + + test("exclude is applied after include", () => { + const s: Scope = { include: ["acme::**"], exclude: ["acme::internal::**"] }; + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme::internal::Secret", s)).toBe(false); + }); + + test("exclude alone narrows the implicit match-everything", () => { + const s: Scope = { exclude: ["acme::internal::**"] }; + expect(match("acme::commerce::Order", s)).toBe(true); + expect(match("acme::internal::Secret", s)).toBe(false); + }); + + test("a bare name with no package is matchable", () => { + expect(match("Order", { include: ["Order"] })).toBe(true); + expect(match("Order", { include: ["*"] })).toBe(true); + }); + + test("regex metacharacters in a pattern are literal", () => { + expect(match("acme::Order.v2", { include: ["acme::Order.v2"] })).toBe(true); + expect(match("acme::OrderXv2", { include: ["acme::Order.v2"] })).toBe(false); + }); + + test("an empty pattern is ERR_SCOPE_PATTERN_INVALID", () => { + let caught: unknown; + try { + compileScope({ include: [""] }); + } catch (err) { + caught = err; + } + expect(caught).toBeDefined(); + expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); + }); + + test("an empty segment is ERR_SCOPE_PATTERN_INVALID", () => { + let caught: unknown; + try { + compileScope({ include: ["acme::::Order"] }); + } catch (err) { + caught = err; + } + expect(caught).toBeDefined(); + expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); + }); + + test("an odd colon run (malformed separator) is ERR_SCOPE_PATTERN_INVALID, not a silently-unmatchable pattern", () => { + // "acme:::Order".split("::") => ["acme", ":Order"] — the leftover ":" + // used to compile as a literal character into `^acme:::Order$`, a + // regex no legal "::"-joined name can ever match. A typo'd include + // pattern therefore silently scoped out EVERYTHING instead of failing + // to load. + let caught: unknown; + try { + compileScope({ include: ["acme:::Order"] }); + } catch (err) { + caught = err; + } + expect(caught).toBeDefined(); + expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); + }); + + test("a single stray colon (not the :: separator) is also ERR_SCOPE_PATTERN_INVALID", () => { + let caught: unknown; + try { + compileScope({ include: ["acme:Order"] }); + } catch (err) { + caught = err; + } + expect(caught).toBeDefined(); + expect(errorCode(caught)).toBe("ERR_SCOPE_PATTERN_INVALID"); + }); +}); diff --git a/server/typescript/packages/sdk/test/source-order.test.ts b/server/typescript/packages/sdk/test/source-order.test.ts new file mode 100644 index 000000000..de8c9bc38 --- /dev/null +++ b/server/typescript/packages/sdk/test/source-order.test.ts @@ -0,0 +1,130 @@ +// server/typescript/packages/sdk/test/source-order.test.ts +// +// The load-ORDER gate for resolved sources. +// +// Order independence (`order-independence.test.ts`) proves the resolved SET +// is a pure function of the declared spec set. It says nothing about the +// order that set is handed to the loader in — and that order IS observable in +// generated output: `codegen-ts`'s barrel emits exports straight from +// `root.objects()` order, and the same order flows into the shared `enums.ts`, +// `meta docs` page ordering, and `meta export`'s `canonicalSerialize` sibling +// order. +// +// The pre-source-resolution toolchain read every file through +// `listMetadataFiles` (metadata-files.ts), which visits FILES at a level before +// descending into that level's subdirectories. A flat lexicographic sort of +// absolute paths disagrees with it the moment a subdirectory name sorts before +// a sibling file — `metaobjects/common/…` before `metaobjects/meta.users.json` +// is exactly that shape, and it is the shape this fixture builds. Every +// `metaobjects/` tree committed in this repository is FLAT, so nothing else +// here can observe the property. +// +// The structural half of the fix is that there is now ONE walker: +// `resolveSources` calls `listMetadataFiles` rather than keeping a second +// recursive walk of its own. These tests pin the resulting order directly, so +// re-splitting the walkers (or "simplifying" either one back to a flat sort) +// goes red rather than silently reordering everyone's generated code. +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { loadMemory } from "../src/memory.js"; +import { DEFAULT_METADATA_DIR, listMetadataFiles } from "../src/metadata-files.js"; +import { resolveSources, type SourceSpec } from "../src/sources.js"; +import { resolveCollection } from "../src/collection.js"; + +let root: string; + +const write = (rel: string, body: object): void => { + const full = join(root, rel); + mkdirSync(dirname(full), { recursive: true }); + writeFileSync(full, JSON.stringify(body), "utf8"); +}; + +const entity = (pkg: string, name: string): object => ({ + "metadata.root": { + package: pkg, + children: [ + { "object.entity": { name, children: [{ "field.string": { name: "id" } }] } }, + ], + }, +}); + +const relative = (files: readonly string[]): string[] => + files.map((f) => f.slice(root.length + 1)); + +beforeEach(() => { + root = mkdtempSync(join(tmpdir(), "metaobjects-source-order-")); + // `common` sorts BEFORE `meta.users.json` as a plain string, so a flat sort + // of absolute paths puts the SUBDIRECTORY first. The walker production used + // before this branch puts the file first. + write(join(DEFAULT_METADATA_DIR, "common", "meta.base.json"), entity("acme", "BaseThing")); + write(join(DEFAULT_METADATA_DIR, "meta.users.json"), entity("acme", "User")); +}); +afterEach(() => { + rmSync(root, { recursive: true, force: true }); +}); + +describe("resolved source order — nested directories", () => { + test("files at a level come before that level's subdirectories", async () => { + const out = await resolveSources(root, [{ path: DEFAULT_METADATA_DIR }]); + expect(relative(out.map((r) => r.file))).toEqual([ + "metaobjects/meta.users.json", + "metaobjects/common/meta.base.json", + ]); + }); + + test("the fixture actually discriminates — a flat sort would order it the other way", async () => { + // Without this the assertion above could pass on a flat-sorting resolver + // and nobody would notice the gate had stopped testing anything. + const out = await resolveSources(root, [{ path: DEFAULT_METADATA_DIR }]); + const files = out.map((r) => r.file); + expect([...files].sort()).not.toEqual(files); + }); + + test("resolveSources agrees with listMetadataFiles, the walker production used before this branch", async () => { + const out = await resolveSources(root, [{ path: DEFAULT_METADATA_DIR }]); + const legacy = await listMetadataFiles(join(root, DEFAULT_METADATA_DIR)); + expect(out.map((r) => r.file)).toEqual(legacy); + }); + + test("resolveCollection's default path resolves that same order", async () => { + const collection = await resolveCollection(root); + const legacy = await listMetadataFiles(join(root, DEFAULT_METADATA_DIR)); + expect([...collection.files]).toEqual(legacy); + }); + + test("the loaded tree's sibling order follows it — the observable half", async () => { + // The reason any of this matters: declaration order survives into + // `root.children()`, which is what the barrel generator emits from. + const collection = await resolveCollection(root); + const loaded = await loadMemory(collection.configDir, { files: collection.files }); + expect(loaded.children().map((c) => c.name)).toEqual(["User", "BaseThing"]); + }); +}); + +describe("resolved source order — across several specs", () => { + beforeEach(() => { + write(join("extra", "nested", "meta.deep.json"), entity("acme", "Deep")); + write(join("extra", "meta.top.json"), entity("acme", "Top")); + }); + + test("each spec contributes its own per-level order, and specs are ordered by content", async () => { + // "extra" sorts before "metaobjects", so its files lead; within each spec + // the per-level rule applies. + const out = await resolveSources(root, [{ path: DEFAULT_METADATA_DIR }, { path: "extra" }]); + expect(relative(out.map((r) => r.file))).toEqual([ + "extra/meta.top.json", + "extra/nested/meta.deep.json", + "metaobjects/meta.users.json", + "metaobjects/common/meta.base.json", + ]); + }); + + test("permuting the specs cannot change the order (set purity survives the per-level walk)", async () => { + const specs: SourceSpec[] = [{ path: DEFAULT_METADATA_DIR }, { path: "extra" }]; + const forward = await resolveSources(root, specs); + const reverse = await resolveSources(root, [...specs].reverse()); + expect(forward).toEqual(reverse); + }); +}); diff --git a/server/typescript/packages/sdk/test/sources.test.ts b/server/typescript/packages/sdk/test/sources.test.ts new file mode 100644 index 000000000..07f503686 --- /dev/null +++ b/server/typescript/packages/sdk/test/sources.test.ts @@ -0,0 +1,143 @@ +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { mkdtempSync, rmSync, mkdirSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { resolveSources, DEFAULT_SOURCES } from "../src/sources.js"; +import { rejectedCode } from "./support/error-code.js"; + +let root: string; +const write = (rel: string, body = "{}") => { + const full = join(root, rel); + mkdirSync(join(full, ".."), { recursive: true }); + writeFileSync(full, body, "utf8"); + return full; +}; + +beforeEach(() => { root = mkdtempSync(join(tmpdir(), "metaobjects-sources-")); }); +afterEach(() => { rmSync(root, { recursive: true, force: true }); }); + +describe("resolveSources", () => { + test("resolves a directory recursively, metadata files only", async () => { + write("model/meta.a.json"); + write("model/nested/meta.b.yaml"); + write("model/notes.txt"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out.map((r) => r.file.replace(root + "/", ""))).toEqual([ + "model/meta.a.json", + "model/nested/meta.b.yaml", + ]); + }); + + test("resolves a single file", async () => { + write("model/meta.a.json"); + const out = await resolveSources(root, [{ path: "model/meta.a.json" }]); + expect(out).toHaveLength(1); + }); + + test("output is canonically sorted regardless of spec order", async () => { + write("b/meta.b.json"); + write("a/meta.a.json"); + const forward = await resolveSources(root, [{ path: "a" }, { path: "b" }]); + const reverse = await resolveSources(root, [{ path: "b" }, { path: "a" }]); + expect(forward.map((r) => r.file)).toEqual(reverse.map((r) => r.file)); + }); + + test("de-duplicates a file contributed by two overlapping specs", async () => { + write("model/meta.a.json"); + const out = await resolveSources(root, [{ path: "model" }, { path: "model/meta.a.json" }]); + expect(out).toHaveLength(1); + }); + + test("the spec attributed to an overlapping file is order-independent, not just the file list", async () => { + write("model/meta.a.json"); + const forward = await resolveSources(root, [{ path: "model" }, { path: "model/meta.a.json" }]); + const reverse = await resolveSources(root, [{ path: "model/meta.a.json" }, { path: "model" }]); + // Deep-equal on the FULL ResolvedSource[] — .spec included, not just .file. + // A first-spec-wins tie-break would pass the two `de-duplicates` / + // `canonically sorted` tests above yet fail here, because which spec is + // attributed would flip between forward and reverse. + expect(forward).toEqual(reverse); + // Pin the actual deterministic winner: content-only comparison picks + // whichever spec's JSON.stringify sorts first, regardless of which was + // declared (or processed) first. + expect(forward).toHaveLength(1); + expect(forward[0]?.spec).toEqual({ path: "model" }); + }); + + test("paths resolve against the config dir, not process.cwd()", async () => { + write("apps/ui/.keep"); + write("model/meta.a.json"); + const out = await resolveSources(join(root, "apps/ui"), [{ path: "../../model" }]); + expect(out).toHaveLength(1); + }); + + test("an unresolvable path is ERR_SOURCE_UNRESOLVED, never a silent skip", async () => { + expect(await rejectedCode(resolveSources(root, [{ path: "missing" }]))).toBe( + "ERR_SOURCE_UNRESOLVED", + ); + }); + + test("resource and package kinds are ERR_SOURCE_KIND_UNSUPPORTED in phase 1", async () => { + expect(await rejectedCode(resolveSources(root, [{ resource: "acme/model" }]))).toBe( + "ERR_SOURCE_KIND_UNSUPPORTED", + ); + expect(await rejectedCode(resolveSources(root, [{ package: "@acme/model" }]))).toBe( + "ERR_SOURCE_KIND_UNSUPPORTED", + ); + }); + + test("an unsupported kind is reported regardless of declaration order relative to an unresolvable path", async () => { + // Kind validation used to be interleaved with per-spec filesystem I/O in + // one loop, so an unsupported-kind spec placed AFTER an unresolvable + // path spec never got reached — the path spec's ERR_SOURCE_UNRESOLVED + // fired first, and the reported code silently depended on which spec + // was declared first. Both orderings must report the SAME code. + const unsupportedFirst: Parameters[1] = [ + { resource: "acme/model" }, + { path: "missing" }, + ]; + const unresolvedFirst: Parameters[1] = [ + { path: "missing" }, + { resource: "acme/model" }, + ]; + expect(await rejectedCode(resolveSources(root, unsupportedFirst))).toBe( + "ERR_SOURCE_KIND_UNSUPPORTED", + ); + expect(await rejectedCode(resolveSources(root, unresolvedFirst))).toBe( + "ERR_SOURCE_KIND_UNSUPPORTED", + ); + }); + + test("_pending is excluded at any depth", async () => { + write("model/meta.a.json"); + write("model/_pending/meta.draft.json"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out).toHaveLength(1); + }); + + test("a nested symlinked directory is followed", async () => { + write("real/meta.b.json"); + write("model/meta.a.json"); + const { symlinkSync } = await import("node:fs"); + symlinkSync(join(root, "real"), join(root, "model/linked"), "dir"); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out).toHaveLength(2); + }); + + test("a dangling symlink inside a source directory is skipped, not a raw ENOENT crash", async () => { + // DirectorySource in @metaobjectsdev/metadata catches and skips exactly + // this case (directory-source.ts). Before the fix, the bare `stat()` in + // collectDir had no try/catch, so a dangling symlink crashed + // resolveSources with a raw Node ENOENT carrying no ERR_ code — on a + // tree the loader itself reads fine. + write("model/meta.a.json"); + const { symlinkSync } = await import("node:fs"); + symlinkSync(join(root, "model/does-not-exist"), join(root, "model/dangling.json")); + const out = await resolveSources(root, [{ path: "model" }]); + expect(out.map((r) => r.file.replace(root + "/", ""))).toEqual(["model/meta.a.json"]); + }); + + test("DEFAULT_SOURCES is the metaobjects/ directory", () => { + expect(DEFAULT_SOURCES).toEqual([{ path: "metaobjects" }]); + }); +}); diff --git a/server/typescript/packages/sdk/test/support/error-code.ts b/server/typescript/packages/sdk/test/support/error-code.ts new file mode 100644 index 000000000..1328fc5be --- /dev/null +++ b/server/typescript/packages/sdk/test/support/error-code.ts @@ -0,0 +1,28 @@ +// server/typescript/packages/sdk/test/support/error-code.ts +// +// Shared by scope.test.ts, sources.test.ts and collection.test.ts. Each used +// to define its own copy of these two helpers, cross-referencing the others +// in a comment as the only sync mechanism — one copy, imported by all three. +// +// Property-based, never message-matching or `instanceof`: a cross-package +// `instanceof ParseError` is silently false when two physical copies of +// `@metaobjectsdev/metadata` are loaded (a globally-installed or linked CLI +// alongside a project-local dependency), so `.code` is the only reliable +// read. + +/** Pull the stable ERR_ code off a caught error, if it carries one. */ +export function errorCode(err: unknown): string { + const code = (err as { code?: unknown }).code; + return typeof code === "string" ? code : "ERR_UNKNOWN"; +} + +/** Await `promise`, expecting it to reject — returns the rejection's stable + * code. */ +export async function rejectedCode(promise: Promise): Promise { + try { + await promise; + } catch (err) { + return errorCode(err); + } + throw new Error("expected the promise to reject, but it resolved"); +}