diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 19d192ba..b111776f 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -13,8 +13,9 @@ model to two destinations at once: Both are generated from the same source document — you never author them separately, and a single `push` keeps them in sync. -This guide covers authoring, deploying, and updating a model. For the Ossie -document format itself, see [ossie.apache.org](https://ossie.apache.org/). +This guide covers authoring, deploying, pulling back, and updating a model. +For the Ossie document format itself, see +[ossie.apache.org](https://ossie.apache.org/). ## Prerequisites @@ -137,6 +138,12 @@ statement runs in the right region; without that permission it falls back to BigQuery's own location inference and warns. Nothing runs under `--validate-only`; add `--print` to see the DDL. +Push to BigQuery is **lossy**: the graph captures the queryable structure — +node tables, edge tables, and measures — but not descriptive metadata +(descriptions, `ai_context`, synonyms, labels), and a metric that does not +reduce to a single MEASURE is dropped with a warning. It is a query surface, +not a copy of your model. + ### What gets created in Knowledge Catalog Each element of your model maps to one catalog resource. Every resource type @@ -154,23 +161,35 @@ An entity entry carries its columns in the `schema` aspect (name, data type, and description per field); a `schema-join` link carries the relationship detail — the paired columns and foreign-key direction — in its aspect. -> **Note — the catalog is not a full copy of your model.** By default the SQL -> expressions are **not** written to Knowledge Catalog: the published system-type -> templates do not yet carry a per-field `semantics` block or a -> `semantic-metric.expression` field, so the default push omits them (pass -> `--emit-expressions` to write them once the templates gain the fields). The -> original vendor SQL (`importedExpression` — e.g. the MAQL or Snowflake form a -> metric was imported from) is never written either. All of it stays in your -> authored document and is still used when generating BigQuery SQL. Keep your -> model document as the source of truth: a model reconstructed only from the -> catalog would come back without its SQL. +> **Note — push to Knowledge Catalog is lossy.** The catalog holds metadata, +> not a full copy of your model. It **stores** names, descriptions, data +> sources, field datatypes and roles, and 1:1 / 1:N relationships (as +> `schema-join` links). By default it does **not** store the SQL expressions: +> the published system-type templates do not yet carry a per-field `semantics` +> block or a `semantic-metric.expression` field, so the default push omits them +> (pass `--emit-expressions` to write the canonical GoogleSQL/ANSI expression +> once the templates gain the fields). It never stores entity keys, `ai_context`, +> field labels, the original vendor SQL (`importedExpression` — e.g. the MAQL or +> Snowflake form a metric was imported from), or M:N relationships. Those stay in +> your authored document (and, for the edges, in the BigQuery property graph); the +> vendor SQL and expressions are still used when generating BigQuery SQL. Keep +> your model document as the source of truth. ## Validation `push` and `--validate-only` run the same checks, **before either destination is touched**, so a model that cannot deploy fails fast instead of half-deploying: -* **Exactly one deployment target per model.** *(static)* +* **Exactly one deployment target per model, and it must be a valid BigQuery + Graph URI.** A model with no target — or with more than one — is rejected, and + so is a single target whose URI does not match + `//bigquery.googleapis.com/projects/

/datasets//propertyGraphs/` (for + example a `propertyGraph`/`propertyGraphs` typo, or a + `…/entryGroups/@bigquery/entries/…` entry form). The error names the offending + URI and the expected form. This gate runs before any destination leg and for + every `--target`, so a malformed target writes **nothing** — not to BigQuery + and **not to Knowledge Catalog**; the push aborts with a non-zero exit and no + entries are created. *(static)* * **Every metric on a BigQuery Graph model resolves to exactly one entity** — otherwise it would be dropped from the BigQuery Graph. Set the metric's attach entity, or scope its expression to a single entity. *(static)* @@ -220,3 +239,109 @@ Every push prints one line per destination summarizing what it did. For a Deployed 1 BigQuery Graph(s). Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. ``` + +## Pull + +`kcmd pull` is the inverse of push's Knowledge Catalog leg: it reads the +`semantic-*` entries back from the catalog and reconstructs local model +documents at `catalog/EntryGroups//.yaml`. Use it to +recover a workspace from a catalog someone else deployed, or to see what the +catalog actually holds. + +```bash +kcmd pull +``` + +Pull reads only from Knowledge Catalog (never BigQuery). Its coordinates come +from the same scope you authored under (`..`). + +| Flag | Effect | +|------|--------| +| `--dry-run` | Reconstruct from the catalog and report what would be written, but write no files. | +| `--model ` | Pull a single model by name; other models in the entry group are left alone. | + +One entry group can hold **many models** — each `semantic-model` entry is a +separate anchor, and pull reconstructs one document per anchor. `--model` +narrows both the fetch and the write to a single anchor. + +Pull writes with the same last-write-wins policy as the core pull: a model that +already exists locally is overwritten in place, and a local-only document (one +with no matching catalog entry) is left untouched — pull never deletes. + +> **Note — pull reconstructs what the catalog holds, not your original file.** +> Pull can only recover what push wrote (see the note under [What gets created in +> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)). What that means in +> practice: +> +> **Recovered exactly** — these come back as authored: +> - Model structure: the model, its entities, and each entity's fields. +> - Field data source and data type. +> - Metrics: name, data type, and attach entity. +> - 1:1 / 1:N relationships: endpoints, foreign-key direction, and join columns +> (from the `schema-join` links). +> - Deployment targets. +> +> **Recovered only if pushed with `--emit-expressions`** — the per-field +> `semantics` block (expressions and the dimension role) and the metric +> expression are omitted from the catalog by default (see the note above), so +> pull returns them only when the push that wrote them used `--emit-expressions`: +> - Field expressions and metric expressions (the canonical GoogleSQL/ANSI form). +> - A field's dimension role, which comes back as a bare `dimension: {}` marker, +> without its detail (`is_time`, and so on). A default push drops the marker +> entirely. +> +> **Recovered, but normalized** — the content survives, the form changes: +> - Relationship *names* come back lowercased/hyphenated (the catalog stores the +> name only in the link id, e.g. `Places Order` → `places-order`). +> - A metric authored with no data type comes back as an explicit `Decimal` +> (push must write a type, and defaults it to `NUMERIC`). +> +> **Not recovered** — push never wrote these, so pull cannot return them: +> - Entity keys / unique keys. +> - `ai_context`. +> - Field labels. +> - The original vendor SQL (`importedExpression`). +> - M:N relationships (the edge lives only in the BigQuery property graph). +> +> **So: a push followed by a pull does not return your original file.** Treat a +> pulled document as a faithful copy of the catalog metadata, not of the authored +> model, and keep the authored document as the source of truth. + +> **Note — writer-side follow-up (not inherent to pull).** One reduction above is +> a limit of what push currently *writes*, not of what pull can recover. It is +> recorded here as a write-side follow-up; the reader (pull) already returns +> everything the catalog holds. +> +> - **Relationship names.** The `schema-join` aspect type's `metadataTemplate` has +> no field for the relationship name, so push cannot store it and pull recovers +> it from the link id — which is lowercased and hyphenated (the entry-link id +> format forbids the original casing/underscores). Returning the name verbatim +> requires adding a name field to the built-in `schema-join` aspect type in +> Knowledge Catalog (server-side), after which the client write/read is trivial; +> it is the same class of gap as the `semantics` field that gates +> `--emit-expressions`. +> +> (A non-canonical deployment target is **not** a pull gap: push rejects it at the +> validation gate before any leg runs, so it is never written — see +> [Validation](#validation).) + +## Permissions + +`push` needs access to whichever destinations you deploy to. + +**BigQuery** — for `--target bq` or `all`, and for the validation pre-flight: + +* `bigquery.jobs.create` in the deployment-target project — to run the deploy's + `CREATE OR REPLACE PROPERTY GRAPH` and the validation dry-run query +* read access on each entity's source table, so the dry-run can resolve it +* `bigquery.datasets.get` on the target dataset (region detection; optional — + push degrades gracefully without it) + +**Knowledge Catalog / Dataplex** — for `--target kc` or `all`: + +* `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group +* `dataplex.entryGroups.useSchemaJoinEntryLink` and + `dataplex.entryGroups.useSchemaJoinAspect` when the model has relationships + +`kcmd pull` needs read access to the same entry group instead — to list its +entries and fetch each `semantic-*` entry with its aspects. diff --git a/toolbox/mdcode/src/libts/layouts/semantic-model.ts b/toolbox/mdcode/src/libts/layouts/semantic-model.ts index 214fe930..42730af7 100644 --- a/toolbox/mdcode/src/libts/layouts/semantic-model.ts +++ b/toolbox/mdcode/src/libts/layouts/semantic-model.ts @@ -91,6 +91,36 @@ export class SemanticModelLayout implements CatalogLayout { return docs; } + // True when a model document with this handle already exists on disk. `pull` + // uses it to report which files it would overwrite vs. create. + hasModel(name: string): boolean { + return fs.existsSync(this.modelPath(name)); + } + + // The absolute path a model document with this handle maps to: + // `/EntryGroups//.yaml`. Path separators in the + // model name are replaced so a name still yields a single flat file. Requires + // the layout to be scoped to an entry group (the semantic-model source always + // is). + modelPath(name: string): string { + if (!this._entryGroup) { + throw new Error( + 'SemanticModel layout has no entry group; cannot resolve a model path.'); + } + const file = `${name.replace(/[/\\]/g, '_')}.yaml`; + return path.join(this._catalogPath, 'EntryGroups', this._entryGroup, file); + } + + // Writes a model's serialized document to its path, creating the EntryGroup + // directory if needed, and indexes it so a later modelDocuments() sees it. + // This is the sink `pull` writes reconstructed models to. + writeModelDocument(name: string, text: string): void { + const localPath = this.modelPath(name); + fs.mkdirSync(path.dirname(localPath), {recursive: true}); + fs.writeFileSync(localPath, text); + this._index.set(name, localPath); + } + // The Knowledge Catalog entry-level members are not applicable to this // push-only layout; the model is authored as a single Ossie document, not as // per-entry Knowledge Catalog files. These are wired when KC-resource emit diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts new file mode 100644 index 00000000..9c863290 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -0,0 +1,480 @@ +// Knowledge Catalog <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// READ direction -- Knowledge Catalog entries -> IR +// (`modelsFromCatalogResources` and its helpers). The WRITE direction (IR -> KC +// entries, `generateCatalogResources`) still lives in `knowledge_catalog.ts` +// and moves here once PR4 (the KC push line) merges, at which point this +// becomes the full two-way converter and `knowledge_catalog.ts` goes away. +// Until then, "where is the KC emit code?" -> `knowledge_catalog.ts`. +// +// TODO(#278): fold the KC WRITE direction in and drop the scaffold. Once +// #278 merges, move `generateCatalogResources` (and the `KcResources` type +// + emit helpers) out of `knowledge_catalog.ts` into this file, delete +// `knowledge_catalog.ts`, and repoint its importers +// (`deploy_knowledge_catalog.ts` and the emitter tests). `idOf` -- shared by +// both directions and re-exported from here only for `pull_kc` today -- becomes +// a plain local. Then this is the sole KC<->IR codec and the SCAFFOLD note +// above comes out. +// +// The reader is the inverse of `generateCatalogResources`: it reconstructs the +// IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). +// `semantic-entity` / `semantic-metric` entries are grouped under their +// `semantic-model` anchor via `parentEntry`; entries of other types are +// ignored. Resources are matched by type-name SUFFIX, so a reader need not know +// which system-type project/location the emitter used. +// +// Fidelity is bounded by what the emitter persisted, so this read is the +// inverse of the WRITE, not of the authored document. It recovers names, +// descriptions, data sources, field datatypes (via the schema aspect), each +// metric's attach entity (re-derived from its expression, as the loader does, +// when the catalog holds one, else the value the emitter persisted), the +// model's deployment targets (from the semantic-model aspect, back into the +// GOOGLE `custom_extensions` block), and 1:1 / 1:N relationships (from the +// `schema-join` entry links a pull fetched -- see +// `modelsFromCatalogResources`'s `entryLinks` argument). The per-field +// `semantics` block -- field/metric expressions and the DIMENSION role -- is +// gated off the catalog by default: the emitter writes it only under +// `--emit-expressions` (see `KcGenerateOptions.emitExpressions`), so those +// three recover only when the push that wrote them enabled it, and a default +// push -> pull drops them. It cannot recover what the emitter never writes: +// entity keys/unique keys, `ai_context`, field labels, +// `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// many-to-many (association) relationships (whose edge lives only in the +// BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads +// back un-typed: the metric aspect persists only `dataType` (both collapse to +// `STRING`, with no metadataType to disambiguate), so -- as with a plain STRING +// field -- the reader leaves it un-typed rather than guess. Relationship NAMES +// come back normalized (lowercased/hyphenated), since the emitter encodes the +// name only in the link id (via `linkSlug`), not in the join aspect. + +import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; + +import {CustomExtension, DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; +import {referencedEntityNames} from './sql_expr_utils'; + +export interface ReadResult { + models: SemanticModel[]; + warnings: string[]; +} + +/** + * Reconstructs the Semantic Model IR from Knowledge Catalog entries. + * + * Returns one model per `semantic-model` anchor plus any warnings (no anchor, + * an orphaned child, an entry missing its aspect data). Entries must already be + * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a + * BASIC list omits aspect data, so the puller re-fetches each entry first. + * + * `entryLinks` are the `schema-join` links a pull fetched for the group (see + * `pull_kc`); each link between two of a model's entity entries is + * reconstructed as a 1:1 / 1:N relationship. Pass `[]` (the default) to + * reconstruct entities and metrics only. + */ +export function modelsFromCatalogResources( + entries: Entry[], entryLinks: EntryLink[] = []): ReadResult { + const warnings: string[] = []; + + const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); + const entityEntries = + entries.filter(e => semanticType(e) === 'semantic-entity'); + const metricEntries = + entries.filter(e => semanticType(e) === 'semantic-metric'); + + if (!anchors.length) { + warnings.push('no semantic-model entry found; nothing to reconstruct'); + return {models: [], warnings: [...new Set(warnings)]}; + } + + // A child belongs to its anchor by parentEntry. When there is exactly one + // anchor, children whose parentEntry does not resolve (e.g. a project-id + // normalization mismatch) are still attached to it rather than dropped. + const anchorNames = new Set(anchors.map(a => a.name)); + const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; + const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( + e => e.parentEntry === anchorName || + (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + + const models = anchors.map(anchor => { + const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + + const entityEntriesForModel = childrenOf(anchor.name, entityEntries); + const entities = entityEntriesForModel.map(e => readEntity(e, warnings)); + const entityNames = entities.map(e => e.name); + const metrics = childrenOf(anchor.name, metricEntries) + .map(e => readMetric(e, entityNames, warnings)); + + // Index this model's entities by their entry id, so a schema-join link can + // resolve its endpoints from the link's entryReferences. The id segment is + // stable across the project-number/id normalization that lookupEntry + // applies to entries but lookupEntryLinks does not apply to link + // references, so matching on it (not the full resource name) keeps + // live-fetched links resolvable. + const entityByEntryId = new Map(); + entityEntriesForModel.forEach( + (e, i) => entityByEntryId.set(idOf(e.name), entities[i])); + + // Relationships come from the schema-join entry links whose two endpoints + // are both this model's entity entries (M:N edges were never published -- + // they live only in the BigQuery property graph -- so stay absent here). + const relationships = + readRelationships(entryLinks, name, entityByEntryId, warnings); + + const model: SemanticModel = {name, entities, relationships, metrics}; + const description = anchor.entrySource?.description; + if (description !== undefined) model.description = description; + // Deployment targets ride back in the same GOOGLE custom_extensions block + // the author wrote them in (the inverse of the emitter's modelAspectData). + const targets = readDeploymentTargets(anchor); + if (targets) model.customExtensions = [targets]; + return model; + }); + + // Flag children that resolved to no anchor at all (only possible with + // multiple anchors, where the sole-anchor fallback does not apply). + if (!soleAnchor) { + for (const child of [...entityEntries, ...metricEntries]) { + if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { + warnings.push(`entry '${ + child.name}' has no resolvable parent semantic-model; omitted`); + } + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// Reconstructs an entity from its `semantic-entity` aspect (the backing source) +// and the built-in `schema` aspect (its fields). Keys are not persisted by the +// emitter and so come back empty. +function readEntity(entry: Entry, warnings: string[]): Entity { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const semantic = aspectData(entry, 'semantic-entity'); + const schema = aspectData(entry, 'schema'); + if (!Object.keys(semantic).length) { + warnings.push(`entity '${ + name}': no semantic-entity aspect data (fetch with the aspect type)`); + } + + const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); + if (!dataSource) { + warnings.push( + `entity '${name}': no backing data source in the semantic-entity ` + + `aspect; 'source' will be empty and the entity may not load`); + } + const entity: Entity = { + name, + dataSource, + keys: [], // not persisted by the emitter; unrecoverable on read + fields: asArray(schema.fields) + .map(fd => readField(fd, name, warnings)) + .filter((f): f is Field => f !== undefined), + }; + const description = entry.entrySource?.description; + if (description !== undefined) entity.description = description; + return entity; +} + + +// Reconstructs a field from one `schema` aspect field record, inverting +// schemaAspectData: the datatype from dataType/metadataType, expressions from +// the nested `semantics` block, and the DIMENSION role back to a dimension +// marker. +function readField(fd: any, entityName: string, warnings: string[]): Field| + undefined { + const name = fd?.name; + if (name === undefined || name === '') { + warnings.push( + `entity '${entityName}': a schema field is missing its name; the ` + + `field is skipped`); + return undefined; + } + const field: Field = {name}; + const sem = fd?.semantics ?? {}; + if (sem.expression !== undefined) field.expression = sem.expression; + const type = irDataType(fd?.dataType, fd?.metadataType); + if (type !== undefined) field.type = type; + if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.description !== undefined) field.description = fd.description; + return field; +} + + +// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` +// is re-derived from the expression (as the loader does) when the aspect holds +// one, else it falls back to the entity the emitter persisted. A default push +// omits the expression entirely (it is gated behind `--emit-expressions`), so +// an absent expression is expected, not an error, and does not warn. +function readMetric( + entry: Entry, entityNames: string[], warnings: string[]): Metric { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + + const metric: Metric = {name}; + if (data.expression !== undefined) metric.expression = data.expression; + const exprForRefs = data.expression ?? ''; + const referenced = referencedEntityNames(exprForRefs, entityNames); + const persistedEntity = + typeof data.entity === 'string' && data.entity !== '' ? data.entity : + undefined; + if (referenced.length === 1) { + // The expression pins exactly one known entity; re-derive it (as the loader + // does) so the attach entity stays consistent with the reconstructed set. + metric.entity = referenced[0]; + } else if (persistedEntity !== undefined) { + // The expression does not pin a single known entity (none, or several), so + // fall back to the attach entity the emitter persisted (metricAspectData) + // rather than dropping it. + metric.entity = persistedEntity; + } else if (exprForRefs && !referenced.length) { + // Parity with the loader's convertMetric: an expression that qualifies no + // known entity is flagged as potentially unplaceable downstream. + warnings.push( + `metric '${name}': expression references no known entity; it may not ` + + `be placeable downstream`); + } + // The emitter writes a required dataType, defaulting a typeless metric to + // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric + // authored without a datatype round-trips as an explicit Decimal rather than + // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) + const type = irDataType(data.dataType, undefined); + if (type !== undefined) metric.type = type; + const description = entry.entrySource?.description; + if (description !== undefined) metric.description = description; + return metric; +} + + +// The inverse of columnDataType/columnMetadataType: maps the schema aspect's +// dataType (disambiguated by metadataType only for the STRING family) back to +// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read +// as un-typed (undefined) -- the loader's default -- since the emitter cannot +// distinguish an authored `String` from an un-typed field (both emit STRING). +function irDataType(dataType: string|undefined, metadataType: string|undefined): + DataType|undefined { + switch (dataType) { + case 'INT64': + return 'Integer'; + case 'NUMERIC': + return 'Decimal'; + case 'FLOAT64': + return 'Float'; + case 'BOOL': + return 'Boolean'; + case 'DATE': + return 'Date'; + case 'TIME': + return 'Time'; + case 'DATETIME': + return 'DateTime'; + case 'TIMESTAMP': + return 'DateTimeTz'; + case 'STRING': + return metadataType === 'OTHER' ? 'Opaque' : undefined; + default: + return undefined; + } +} + + +// The inverse of resourcePath: a BigQuery linked-resource URI becomes the +// canonical `project.dataset.table` string; anything else (a verbatim query or +// a passthrough reference) is returned unchanged. +function dataSourceFromResource(resource: string|undefined): string { + const value = (resource ?? '').trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + + +// The bare `semantic-*` type of an entry, matched by entryType suffix so the +// system-type project/location need not be known. Returns undefined for entries +// that are not part of a semantic model. +function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| + 'semantic-metric'|undefined { + for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as + const) { + if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; + } + return undefined; +} + + +// The `data` payload of an entry's aspect of the given bare type. See +// aspectDataOf; entries and entry links carry aspects in the same shape. +function aspectData(entry: Entry, type: string): Record { + return aspectDataOf(entry.aspects, type); +} + + +// The `data` payload of an aspect of the given bare type from an aspect map, +// matched by the aspect key's `.` suffix or the aspectType's +// `/aspectTypes/` suffix (robust to whichever system-type +// project/location the emitter used). Returns an empty object when the aspect +// is absent. +function aspectDataOf(aspects: Record|undefined, type: string): + Record { + for (const [key, aspect] of Object.entries(aspects ?? {})) { + if (key.endsWith(`.${type}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + + +// Recovers the model's deployment targets from its semantic-model aspect back +// into the GOOGLE custom_extension the author declared them in (the inverse of +// the emitter's modelAspectData). Returns undefined when the model has none. +function readDeploymentTargets(anchor: Entry): CustomExtension|undefined { + const targets = + asArray(aspectData(anchor, 'semantic-model').deploymentTargets) + .filter((t): t is string => typeof t === 'string' && t !== ''); + if (!targets.length) return undefined; + return { + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }; +} + + +// Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link +// whose two endpoints are both this model's entity entries becomes one +// direct-FK relationship. +// +// Endpoint identity comes from the link's entryReferences (matched by entry id +// via `entityByEntryId`), NOT from the aspect's table names: the id is unique +// per entity (two entities can share a table) and survives the +// project-number/id normalization lookupEntry applies to entries but +// lookupEntryLinks does not apply to link references. The aspect then supplies +// the join columns and the direction -- its `source` side is the foreign-key +// side, named by its data source -- used only to orient the two endpoints. The +// link id encodes the (normalized) relationship name. Links are deduped -- +// schema-join is undirected, so a per-entry lookup can return the same link +// once from each endpoint. +function readRelationships( + links: EntryLink[], modelName: string, entityByEntryId: Map, + warnings: string[]): Relationship[] { + const prefix = linkNamePrefix(modelName); + const seen = new Set(); + const out: Relationship[] = []; + for (const link of links) { + if (!link.entryLinkType?.endsWith('/entryLinkTypes/schema-join')) continue; + const refs = link.entryReferences ?? []; + if (refs.length !== 2) continue; + // Resolve both endpoints from the link's references. A reference that is + // not one of this model's entities (another model, or a non-entity) means + // the link does not belong here -- skip it quietly, as with M:N edges. + const endA = entityByEntryId.get(idOf(refs[0].name)); + const endB = entityByEntryId.get(idOf(refs[1].name)); + if (!endA || !endB) continue; + + const key = linkDedupKey(link); + if (seen.has(key)) continue; + seen.add(key); + + const join = asArray(aspectDataOf(link.aspects, 'schema-join').joins)[0]; + if (!join) { + warnings.push( + `entry link '${link.name}': no schema-join aspect data; the ` + + `relationship is skipped`); + continue; + } + // Direction lives in the aspect: `source` is the foreign-key side, named by + // its data source. Orient the two endpoints by matching that name, keeping + // join.source.fields paired with the source side. When neither orientation + // matches -- or both do, because the endpoints share a data source -- the + // direction is undecidable, so keep the reference order and warn rather + // than drop the edge. + const srcName = (join.source?.name ?? '').trim(); + const tgtName = (join.target?.name ?? '').trim(); + const aSrc = (endA.dataSource ?? '').trim(); + const bSrc = (endB.dataSource ?? '').trim(); + const endAIsSource = aSrc === srcName && bSrc === tgtName; + const endBIsSource = bSrc === srcName && aSrc === tgtName; + let [source, destination] = [endA, endB]; + if (endBIsSource && !endAIsSource) { + [source, destination] = [endB, endA]; + } else if (!endAIsSource && !endBIsSource) { + warnings.push( + `entry link '${link.name}': join direction does not match either ` + + `endpoint's data source; using the reference order`); + } else if (endAIsSource && endBIsSource) { + warnings.push( + `entry link '${link.name}': join direction is ambiguous (endpoints ` + + `share a data source); using the reference order`); + } + const rel: Relationship = { + name: relationshipName(link.name, prefix), + source: {entity: source.name, columns: asArray(join.source?.fields)}, + destination: + {entity: destination.name, columns: asArray(join.target?.fields)}, + }; + if (join.description !== undefined) rel.description = join.description; + out.push(rel); + } + return out; +} + + +// A stable dedup key for an entry link: its name when present, else its +// (order-independent) endpoint pair and type. Guards against a per-endpoint +// lookup returning an unnamed link once from each of its two endpoints. +export function linkDedupKey(link: EntryLink): string { + if (link.name) return link.name; + const refs = (link.entryReferences ?? []).map(r => r.name).sort(); + return `${refs.join('')}${link.entryLinkType ?? ''}`; +} + + +// Best-effort recovery of the relationship name from the link id, which the +// emitter built as linkSlug(`-`) (lowercased/hyphenated -- see +// knowledge_catalog.ts). Strips the model prefix when present; the name comes +// back normalized, never verbatim. +function relationshipName( + linkName: string|undefined, modelPrefix: string): string { + const id = idOf(linkName ?? ''); + if (modelPrefix && id.startsWith(`${modelPrefix}-`)) { + const rest = id.slice(modelPrefix.length + 1); + if (rest) return rest; + } + return id; +} + + +// Mirrors the model-name portion of the emitter's linkSlug so the prefix a link +// id was built with can be stripped. Kept local rather than imported: this +// read-side module must not depend on the write-side emitter. Exported for the +// symmetry test, which reuses it to reproduce the normalized relationship name +// rather than reimplementing the slug rule. +// +// It reproduces linkSlug's normalization AND its 63-char cap + trailing-hyphen +// re-strip, so the prefix stays aligned with the id even when the combined +// `-` slug was truncated. (linkSlug's `|| 'link'` empty-slug +// fallback is intentionally omitted: an empty prefix strips nothing and +// relationshipName then returns the id verbatim, which is already correct. When +// the cap truncates into the model slug itself the relationship name is +// unrecoverable regardless -- relationshipName returns the truncated id.) +export function linkNamePrefix(modelName: string): string { + return modelName.toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^[^a-z]+/, '') + .replace(/^-+|-+$/g, '') + .slice(0, 63) + .replace(/-+$/, ''); +} + + +// The id segment of a full entry resource name (after the last '/'). +export function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} diff --git a/toolbox/mdcode/src/libts/semantic/osi_converter.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts new file mode 100644 index 00000000..b8b260aa --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -0,0 +1,270 @@ +// OSI (open, AI-first semantics) <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// WRITE direction -- IR -> OSI YAML (`serializeModel`). The READ direction +// (OSI YAML -> IR) still lives in `loader.ts` and moves here once PR4 (the +// KC push line) merges, at which point this becomes the full two-way +// converter. Until then, "where is the OSI parser?" -> `loader.ts`. +// +// TODO(#278): fold the OSI READ direction in and drop the scaffold. Once +// #278 merges, move `loadModels` / `fromDocument` / `loadSemanticModels` +// (and their Load* types) out of `loader.ts` into this file, delete +// `loader.ts`, and repoint its importers (`src/tool/commands.ts` and the +// load / validate / bigquery tests). Then this is the sole OSI<->IR codec +// and the SCAFFOLD note above comes out. +// +// Serializing is the inverse of `loader.ts`: `loader` reads authored YAML +// into the IR; this module writes the IR out as a YAML document the loader +// can read back. It is the local-workspace sink for `pull` (Knowledge +// Catalog -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR +// -> a destination. +// +// Fidelity is at the IR level, not byte-for-byte with a hand-authored file. The +// loader normalizes several authoring conveniences into the IR at load time, so +// they are already gone before serialization and cannot be reproduced here: +// * per-dialect `expression.dialects[]` variants collapse to at most two +// forms +// (a target/canonical `expression` + an `importedExpression`); only those +// are re-emitted, each under a single dialect label. +// * comments and key ordering are not preserved. +// What the loader DOES keep on the IR round-trips here: names, descriptions, +// `ai_context` (instructions / synonyms / examples), `custom_extensions` +// (verbatim), keys and unique keys, data sources, field datatypes / labels / +// dimension flags, expressions, and relationship join columns. See +// serialize.test.ts. +// +// An `association` (junction-table) relationship has no open-format syntax (the +// loader cannot produce one), so only its direct foreign-key view (from/to + +// columns) is serialized; the junction detail is dropped with a note. + +import * as yaml from 'yaml'; + +import {AiContext, CustomExtension, Entity, Field, Metric, Relationship, SemanticModel,} from './ir'; + +// The schema version the loader was written against; re-emitted verbatim so a +// serialized document loads without a version-mismatch warning. Mirrors +// loader.SUPPORTED_VERSION. +const SERIALIZED_VERSION = '0.2.0.dev0'; + +// The dialect label for the IR's target/canonical `expression`. The IR does not +// record which authored dialect that string came from (the loader picked it +// from the target dialect or the ANSI_SQL fallback and discarded the label), +// and its contract is "GoogleSQL-valid". BIGQUERY is the loader's default +// target dialect, so labeling the canonical form BIGQUERY makes the loader +// re-pick it exactly on reload -- a clean round trip with no dialect-fallback +// note. +const CANONICAL_DIALECT = 'BIGQUERY'; + +// The dialect label used for an `importedExpression` whose `importedDialect` +// was lost (e.g. read back from a Knowledge Catalog aspect that does not +// persist the dialect). A non-target, non-canonical label so the loader +// re-reads it as the imported vendor form rather than the canonical expression. +const UNKNOWN_IMPORTED_DIALECT = 'IMPORTED'; + +export interface SerializeResult { + yaml: string; + warnings: string[]; +} + +/** + * Serializes a single semantic model to a YAML document string in the open + * AI-first semantics format. The document contains exactly this one model; + * `pull` writes one file per model + * (catalog/EntryGroups//.yaml). + * + * Warnings flag IR content that has no loadable representation (an association + * relationship's junction detail), so the caller can surface the lossy edge. + */ +export function serializeModel(model: SemanticModel): SerializeResult { + const warnings: string[] = []; + const text = yaml.stringify(modelDocument(model, warnings)); + return {yaml: text, warnings: [...new Set(warnings)]}; +} + +// Builds the plain document object (version + one model) that yaml.stringify +// renders. Kept separate so tests can assert the structure without parsing +// YAML. +export function modelDocument( + model: SemanticModel, warnings: string[] = []): Record { + return { + version: SERIALIZED_VERSION, + semantic_model: [modelDoc(model, warnings)], + }; +} + +function modelDoc( + model: SemanticModel, warnings: string[]): Record { + // `datasets` is required (min 1) by the loader. A reconstructed model with no + // entities (e.g. every entity fetch failed during a pull) would serialize to + // a document the loader rejects; emit the (empty) array but flag it so the + // lossy edge is visible rather than surfacing later as an opaque load error. + const datasets = (model.entities ?? []).map(e => datasetDoc(e, warnings)); + if (!datasets.length) { + warnings.push( + `model '${model.name}': no datasets (entities); the document requires ` + + `at least one and will not load until an entity is present.`); + } + return compact({ + name: model.name, + description: model.description, + ai_context: aiContextDoc(model.aiContext), + custom_extensions: customExtensionsDoc(model.customExtensions), + datasets, + relationships: nonEmpty( + (model.relationships ?? []).map(r => relationshipDoc(r, warnings))), + metrics: nonEmpty((model.metrics ?? []).map(m => metricDoc(m, warnings))), + }); +} + +function datasetDoc( + entity: Entity, warnings: string[]): Record { + return compact({ + name: entity.name, + source: entity.dataSource, + primary_key: nonEmpty(entity.keys), + unique_keys: nonEmpty(entity.uniqueKeys), + description: entity.description, + ai_context: aiContextDoc(entity.aiContext), + fields: nonEmpty((entity.fields ?? []).map(f => fieldDoc(f, warnings))), + custom_extensions: customExtensionsDoc(entity.customExtensions), + }); +} + +function fieldDoc(field: Field, warnings: string[]): Record { + const expression = expressionDoc( + field.expression, field.importedExpression, field.importedDialect); + if (!expression) { + warnings.push( + `field '${field.name}': no expression; the loader requires one per ` + + `field and the document will not load until it is set.`); + } + return compact({ + name: field.name, + expression, + datatype: field.type, + label: field.label, + dimension: dimensionDoc(field), + description: field.description, + ai_context: aiContextDoc(field.aiContext), + custom_extensions: customExtensionsDoc(field.customExtensions), + }); +} + +function metricDoc(metric: Metric, warnings: string[]): Record { + // `entity` is derived by the loader from the expression's entity qualifiers, + // so it is intentionally not emitted: the loader recomputes it on reload. + const expression = expressionDoc( + metric.expression, metric.importedExpression, metric.importedDialect); + if (!expression) { + warnings.push( + `metric '${metric.name}': no expression; the loader requires one per ` + + `metric and the document will not load until it is set.`); + } + return compact({ + name: metric.name, + expression, + datatype: metric.type, + description: metric.description, + ai_context: aiContextDoc(metric.aiContext), + custom_extensions: customExtensionsDoc(metric.customExtensions), + }); +} + +// Inverts loader.convertRelationship: `from`/`to` are the endpoint entities and +// `from_columns`/`to_columns` are their positional join columns. An association +// (junction-table) edge has no open-format syntax, so only this direct-FK view +// is emitted and the junction detail is flagged. +function relationshipDoc( + rel: Relationship, warnings: string[]): Record { + if (rel.association) { + warnings.push( + `relationship '${ + rel.name}': association (junction-table) detail has no ` + + `open-format representation and is not serialized; only its foreign-key ` + + `endpoints are written.`); + } + return compact({ + name: rel.name, + from: rel.source.entity, + to: rel.destination.entity, + from_columns: nonEmpty(rel.source.columns), + to_columns: nonEmpty(rel.destination.columns), + description: rel.description, + ai_context: aiContextDoc(rel.aiContext), + custom_extensions: customExtensionsDoc(rel.customExtensions), + }); +} + +// Renders the `expression` object matching the loader's schema (a `dialects` +// array of {dialect, expression}). Emits the target/canonical form under +// CANONICAL_DIALECT and the imported vendor form under its own dialect, so the +// loader re-picks each into the same IR field. Returns undefined when neither +// form is present (the loader requires an expression, but a pathological field +// with none is dropped rather than fabricated). +function expressionDoc( + expression: string|undefined, importedExpression: string|undefined, + importedDialect: string|undefined): Record|undefined { + const dialects: {dialect: string; expression: string}[] = []; + if (expression !== undefined) { + dialects.push({dialect: CANONICAL_DIALECT, expression}); + } + if (importedExpression !== undefined) { + let label = importedDialect ?? UNKNOWN_IMPORTED_DIALECT; + // Never emit two dialect entries under the same label: the loader would + // pick between them non-deterministically. If the imported form's dialect + // collides with the canonical label already pushed, fall back to the + // imported placeholder. + if (expression !== undefined && label === CANONICAL_DIALECT) { + label = UNKNOWN_IMPORTED_DIALECT; + } + dialects.push({dialect: label, expression: importedExpression}); + } + return dialects.length ? {dialects} : undefined; +} + +// Emits the field's `dimension` block when present, inverting +// loader.convertField (which sets `dimension = {}` for a bare marker and copies +// `is_time`). The key is emitted whenever the IR carries dimension metadata, +// even for an empty marker, so a dimension field reloads as a dimension. +function dimensionDoc(field: Field): Record|undefined { + if (!field.dimension) return undefined; + return compact({is_time: field.dimension.isTime}); +} + +// Emits the structured `ai_context` (the only authoring path the loader reads +// synonyms/instructions/examples from), inverting loader.normalizeAiContext. +// Returns undefined when the context carries nothing, so `compact` drops the +// key. +function aiContextDoc(ai: AiContext|undefined): Record|undefined { + if (!ai) return undefined; + const doc = compact({ + instructions: ai.instructions, + synonyms: nonEmpty(ai.synonyms), + examples: nonEmpty(ai.examples), + }); + return Object.keys(doc).length ? doc : undefined; +} + +// Emits vendor `custom_extensions` verbatim (`vendorName` -> `vendor_name`), +// inverting loader.toCustomExtensions. Returns undefined when there are none. +function customExtensionsDoc(exts: CustomExtension[]|undefined): + Record[]|undefined { + if (!exts || !exts.length) return undefined; + return exts.map(e => ({vendor_name: e.vendorName, data: e.data})); +} + +// Drops undefined-valued keys so the emitted YAML only shows fields the model +// actually set (matching the emitters' `compact` convention). +function compact>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +// Returns the array, or undefined when empty/absent, so an empty list is +// omitted rather than rendered as `[]`. +function nonEmpty(items: T[]|undefined): T[]|undefined { + return items && items.length ? items : undefined; +} diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts new file mode 100644 index 00000000..a1ebae76 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -0,0 +1,241 @@ +// Pull: fetch a semantic model from a live Knowledge Catalog into the IR. +// +// The read-direction counterpart of `deployKnowledgeCatalog` (push, in +// `deploy_knowledge_catalog.ts`). Unlike a write, a pull needs no server-side +// type provisioning -- only that the `semantic-*` entries exist. It enumerates +// the entry group, keeps the semantic entries, hydrates each one's aspect data +// (a BASIC list omits aspect data, so each entry is re-fetched with its aspect +// types -- an entity needs BOTH its `semantic-entity` aspect and the built-in +// `schema` aspect), and hands the hydrated entries to the pure reader +// (`kc_converter.modelsFromCatalogResources`). +// +// TODO(#278): Layer 2 push/pull symmetry. This is the pull half over the +// pure `kc_converter` codec; the push half (`deployKnowledgeCatalog`) still +// lives in `deploy_knowledge_catalog.ts`. Once #278 merges, rename that +// file to `push_kc.ts` (repointing `src/tool/commands.ts` and its test) so +// the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no +// logic moves. + +import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; + +import {SemanticModel} from './ir'; +import {idOf, linkDedupKey, modelsFromCatalogResources} from './kc_converter'; + +export interface KcPullOptions { + project: string; + location: string; + entryGroup: string; + model?: string; // limit to a single model by name (default: all) +} + +export interface KcPullResult { + models: SemanticModel[]; + warnings: string[]; +} + +// Upper bound on in-flight aspect-hydration fetches during a pull. +const HYDRATE_CONCURRENCY = 8; + +// Reads the semantic models back from a Knowledge Catalog entry group. Emits no +// console output; warnings (skipped entries, no match for --model, reader +// warnings) are returned for the caller to print. +export async function pullKnowledgeCatalog( + cat: CatalogClient, opts: KcPullOptions): Promise { + const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const warnings: string[] = []; + + // Enumerate the group (paging is inherently sequential) and pick the semantic + // entries, then hydrate their aspects concurrently: a BASIC list omits aspect + // data, so each entry needs its own lookupEntry, and those fetches are + // independent. The pool preserves input order so warnings stay deterministic. + const targets: {entry: Entry; aspectTypes: string[]}[] = []; + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const aspectTypes = semanticAspectTypes(entry.entryType); + if (aspectTypes) targets.push({entry, aspectTypes}); + // else: not part of a semantic model; ignore it. + } + + // When scoped to one model, hydrate only that model's entries -- its anchor + // (matched by name) plus the children pointing at it. A list already carries + // entrySource + parentEntry, so this avoids fetching every other model's + // aspects. No match short-circuits with just the not-found warning. + let scoped = targets; + if (opts.model) { + scoped = scopeToModel(targets, opts.model); + if (!scoped.length) { + return { + models: [], + warnings: + [`no semantic model named '${opts.model}' found in ${destination}`], + }; + } + } + + const fetched = await mapConcurrent( + scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry '${entry.name}' (status ${ + res.status}); skipped` + }; + } + return {entry: res.result}; + }); + + const hydrated: Entry[] = []; + for (const r of fetched) { + if (r.entry) + hydrated.push(r.entry); + else if (r.warning) + warnings.push(r.warning); + } + + // Second fetch pass: relationships are schema-join entry links, which the + // entry list/lookup does not return. The catalog exposes links only per + // referenced entry (:lookupEntryLinks), so fan out over the entity entries + // and dedup (linkDedupKey) -- schema-join is undirected, so each link comes + // back once from each of its two endpoints. + const entityEntries = hydrated.filter( + e => e.entryType?.endsWith('/entryTypes/semantic-entity')); + const linkResults = + await mapConcurrent(entityEntries, HYDRATE_CONCURRENCY, async entry => { + const linkType = schemaJoinLinkType(entry.entryType); + const res = await cat.lookupEntryLinks(opts.project, opts.location, { + entry: entry.name, + entryLinkTypes: linkType ? [linkType] : undefined, + }); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry links for '${entry.name}' (status ${ + res.status}); relationships may be incomplete`, + }; + } + return {links: res.result}; + }); + + const seenLinks = new Set(); + const entryLinks: EntryLink[] = []; + for (const r of linkResults) { + if (r.warning) { + warnings.push(r.warning); + continue; + } + for (const link of r.links ?? []) { + const key = linkDedupKey(link); + if (seenLinks.has(key)) continue; + seenLinks.add(key); + entryLinks.push(link); + } + } + + const read = modelsFromCatalogResources(hydrated, entryLinks); + warnings.push(...read.warnings); + + // Defense in depth: keep only the requested model even if the reader surfaced + // another anchor (e.g. a child whose parentEntry pointed outside the scope). + let models = read.models; + if (opts.model) { + models = models.filter(m => m.name === opts.model); + if (!models.length) { + warnings.push( + `no semantic model named '${opts.model}' found in ${destination}`); + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// The aspect type resource names to hydrate for a semantic entry, derived from +// its entryType (the aspect types are the parallel resources in the same +// project/location). An entity carries two aspects: its `semantic-entity` +// aspect and the built-in `schema` aspect that holds its fields. Returns +// undefined for entries that are not part of a semantic model. +function semanticAspectTypes(entryType: string): string[]|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + const typeBase = entryType.slice(0, idx); + const t = entryType.slice(idx + marker.length); + const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + switch (t) { + case 'semantic-model': + return [aspectType('semantic-model')]; + case 'semantic-entity': + return [aspectType('semantic-entity'), aspectType('schema')]; + case 'semantic-metric': + return [aspectType('semantic-metric')]; + default: + return undefined; + } +} + + +// The schema-join entry link type resource, derived from an entity's entryType +// base (the link type is the parallel resource in the same project/location the +// emitter referenced). Used to filter :lookupEntryLinks to just the +// relationship links. Returns undefined for an entryType with no recognizable +// base. +function schemaJoinLinkType(entryType: string): string|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + return `${entryType.slice(0, idx)}/entryLinkTypes/schema-join`; +} + + +// Restricts hydration targets to a single model: the semantic-model anchor +// whose name (entrySource.displayName, else the entry id) matches `model`, plus +// every child entry whose parentEntry is that anchor. Uses only list-level +// fields (no aspect data), so it runs before hydration and avoids fetching +// unrelated models' aspects. Returns [] when no anchor matches. +function scopeToModel( + targets: {entry: Entry; aspectTypes: string[]}[], + model: string): {entry: Entry; aspectTypes: string[]}[] { + const isAnchor = (t: {entry: Entry}) => + !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); + const allAnchorNames = + new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = + new Set(targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? + idOf(t.entry.name)) === model) + .map(t => t.entry.name)); + if (!matchedAnchorNames.size) return []; + // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds + // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a + // project-id normalization mismatch) still belongs to it. Without this a + // scoped pull would drop children a full pull keeps. + const soleAnchor = + allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; + const soleMatched = + soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); + return targets.filter( + t => matchedAnchorNames.has(t.entry.name) || + matchedAnchorNames.has(t.entry.parentEntry ?? '') || + (soleMatched && !isAnchor(t) && + !allAnchorNames.has(t.entry.parentEntry ?? ''))); +} + + +// Maps `items` through `fn` with at most `limit` calls in flight, returning +// results in input order (so downstream ordering stays deterministic). +async function mapConcurrent( + items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker(): Promise { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + } + const workers = + Array.from({length: Math.min(limit, items.length)}, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index bd8cf696..1c67d76f 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -14,6 +14,8 @@ import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; import {BigQueryClient} from '../libts/gcp/bigquery'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; +import {pullKnowledgeCatalog} from '../libts/semantic/pull_kc'; +import {serializeModel} from '../libts/semantic/osi_converter'; import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; @@ -158,15 +160,20 @@ export async function init(options: InitOptions): Promise { } -export async function pull(): Promise { +export interface PullOptions { + // Reconstruct + report only; never writes a file. Mirrors push --validate-only. + dryRun?: boolean; + // Limit the pull to a single model by name (default: all in the entry group). + model?: string; +} + + +export async function pull(options: PullOptions = {}): Promise { const ctx = context.ApiContext.default(); const snapshot = await kcmd.CatalogSnapshot.fromPath('.', ctx); if (snapshot.manifest.source.type === Sources.SEMANTIC_MODEL) { - console.log( - 'Semantic-model scope: nothing to pull. Knowledge Catalog resource ' + - 'pull for the semantic model is not yet implemented.'); - return 0; + return await pullSemanticModel(ctx, snapshot, options); } const catalog = new dataplex.CatalogClient(ctx); @@ -360,3 +367,77 @@ async function pushKnowledgeCatalog( unlinked}.`); return 0; } + + +// Pulls the semantic model's Knowledge Catalog entries back into local model +// documents (catalog/EntryGroups//.yaml) and prints the +// result. The destination coordinates come from the scope +// (project.location.entryGroup). Overwrite policy matches the core pull: +// last-write-wins, local-only documents are left untouched (never deleted). +// Returns a process exit code (0 on success). +async function pullSemanticModel( + ctx: context.ApiContext, snapshot: kcmd.CatalogSnapshot, + options: PullOptions): Promise { + // The semantic-model source always resolves to the SemanticModel layout + // (see createLayout), so these casts are safe. + const layout = snapshot.layout as SemanticModelLayout; + const source = snapshot.manifest.source as SemanticModelSource; + + console.log(options.dryRun + ? 'Reconstructing semantic model from Knowledge Catalog (dry run)...' + : 'Pulling semantic model from Knowledge Catalog...'); + + const catalog = new dataplex.CatalogClient(ctx); + const result = await pullKnowledgeCatalog(catalog, { + project: source.project, + location: source.location, + entryGroup: source.entryGroup, + model: options.model, + }); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + + if (!result.models.length) { + console.log('No semantic models found; nothing to pull.'); + return 0; + } + + let created = 0; + let updated = 0; + // Guard against two reconstructed models whose names map to the same file + // (path-separator sanitizing, or two anchors sharing a display name): the + // later write would silently clobber the earlier. Track written paths so the + // collision is reported and the dry-run/real counts agree on the repeat. + const writtenBy = new Map(); + for (const model of result.models) { + const serialized = serializeModel(model); + for (const w of serialized.warnings) { + console.warn(`Warning: [${model.name}] ${w}`); + } + const target = layout.modelPath(model.name); + const prior = writtenBy.get(target); + if (prior !== undefined && prior !== model.name) { + console.warn( + `Warning: models '${prior}' and '${model.name}' both map to ` + + `${target}; the later overwrites the earlier -- rename one model.`); + } + const existed = writtenBy.has(target) || layout.hasModel(model.name); + writtenBy.set(target, model.name); + if (options.dryRun) { + console.log(` would ${existed ? 'update' : 'create'} ${target}`); + } + else { + layout.writeModelDocument(model.name, serialized.yaml); + console.log(` ${existed ? 'updated' : 'created'} ${target}`); + } + if (existed) updated++; + else created++; + } + + console.log(options.dryRun + ? `Dry run: would write ${created} new and ${updated} updated model document(s).` + : `Wrote ${created} new and ${updated} updated model document(s).`); + return 0; +} diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index cbc67e95..84cceff9 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -25,10 +25,12 @@ cli.command('init', 'Initialize a new catalog snapshot') cli.command('pull', 'Pull catalog entries') - .action(async () => { + .option('--dry-run', 'Reconstruct and report only; do not write files (semantic-model scope)') + .option('--model ', 'Limit the pull to a single model by name (semantic-model scope)') + .action(async (options) => { let exitCode = 1; try { - exitCode = await commands.pull(); + exitCode = await commands.pull(options); } catch (err: any) { console.error('Error:', err.message || err); diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml new file mode 100644 index 00000000..ffdfa798 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml @@ -0,0 +1,29 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: + - name: orders + source: demo.sales.orders + primary_key: + - o_orderkey + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml new file mode 100644 index 00000000..f966f619 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -0,0 +1,16 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: + - name: orders + source: demo.sales.orders + fields: + - name: o_orderkey + - name: o_totalprice + metrics: + - name: total_revenue + datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml new file mode 100644 index 00000000..40c22466 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml @@ -0,0 +1,86 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + ai_context: + instructions: Use this model for order analysis. + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' + datasets: + - name: orders + source: samples.tpch.orders + primary_key: + - o_orderkey + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + label: Order Date + dimension: + is_time: true + ai_context: + synonyms: + - order date + - date + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + - name: customer + source: samples.tpch.customer + primary_key: + - c_custkey + fields: + - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + description: Total order revenue + ai_context: + synonyms: + - revenue + - sales + - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml new file mode 100644 index 00000000..ceb6bed1 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -0,0 +1,39 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' + datasets: + - name: orders + source: samples.tpch.orders + description: One row per order + fields: + - name: o_orderkey + description: Order identifier + - name: o_custkey + - name: o_orderdate + - name: o_totalprice + - name: customer + source: samples.tpch.customer + fields: + - name: c_custkey + - name: c_name + description: Customer name + relationships: + - name: orders-to-customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey + metrics: + - name: total_revenue + datatype: Decimal + description: Total order revenue + - name: order_count + datatype: Decimal + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml new file mode 100644 index 00000000..27acf1e6 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml @@ -0,0 +1,218 @@ +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + primary_key: + - ss_item_sk + - ss_ticket_number + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_item_sk}" + dimension: + is_time: false + - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_ticket_number}" + dimension: + is_time: false + - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_customer_sk}" + dimension: + is_time: false + - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_store_sk}" + dimension: + is_time: false + - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_quantity}" + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_sales_price}" + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_ext_sales_price}" + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_net_profit}" + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + primary_key: + - c_customer_sk + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: + is_time: false + - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: + is_time: false + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: + is_time: false + description: Customer last name + - name: item + source: tpcds.public.item + primary_key: + - i_item_sk + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: + is_time: false + - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: + is_time: false + - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: + is_time: false + - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + primary_key: + - s_store_sk + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: + is_time: false + - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: + is_time: false + - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: + is_time: false + - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: + is_time: false + - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes + custom_extensions: + - vendor_name: GOODDATA + data: '{"date_dimension": true, "granularities": ["DAY", "WEEK", "MONTH", + "QUARTER", "YEAR"]}' + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml new file mode 100644 index 00000000..ab45bace --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -0,0 +1,82 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + - name: ss_ticket_number + - name: ss_customer_sk + - name: ss_store_sk + - name: ss_quantity + description: Quantity of items sold + - name: ss_sales_price + description: Sales price per unit + - name: ss_ext_sales_price + description: Extended sales price (quantity * price) + - name: ss_net_profit + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + - name: c_first_name + description: Customer first name + - name: c_last_name + description: Customer last name + - name: item + source: tpcds.public.item + description: Item/Product dimension + fields: + - name: i_item_sk + - name: i_brand + - name: i_category + - name: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + description: Store dimension with location attributes + fields: + - name: s_store_sk + - name: s_store_name + - name: s_city + - name: s_state + - name: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes + relationships: + - name: store-sales-to-date-dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store-sales-to-customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store-sales-to-item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store-sales-to-store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts new file mode 100644 index 00000000..4ccac618 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -0,0 +1,793 @@ +// Behavior specification for the KC converter's read direction +// (modelsFromCatalogResources in src/libts/semantic/kc_converter.ts). +// +// The reader is the inverse of the emitter (generateCatalogResources). The +// central guarantee is an emitter -> reader round trip: emit a model's entries +// AND entry links, read them back, and get an IR equal to the source WHERE the +// emitter is lossless. The write drops content by design (entity keys, +// ai_context, field labels, importedDialect, and many-to-many relationships -- +// see the emitter header), so the expected read-back is the source model with +// exactly those fields cleared. It ALSO drops the per-field `semantics` block +// (field/metric expressions and the DIMENSION role) unless the push enabled +// `emitExpressions`, so a DEFAULT round trip (roundTrip) loses those too, while +// a full round trip (roundTripFull, emitExpressions: true) keeps them. 1:1 / +// 1:N relationships and deployment targets DO round-trip either way (via +// schema-join links and the semantic-model aspect), except that relationship +// names come back normalized (lowercased/hyphenated). Targeted tests pin the +// mapping details a round trip cannot isolate (the dataType inverse, the +// DIMENSION role, resource-URI parsing, metric attach re-derivation, +// relationship endpoint/direction recovery, and parent/anchor grouping). + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; + +import {CustomExtension, Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {linkNamePrefix, modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {loadModels} from '../../../src/libts/semantic/loader'; +import {serializeModel} from '../../../src/libts/semantic/osi_converter'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +// Emits a model to entries + entry links and reads it straight back. The +// default push omits the per-field `semantics` block (expressions + role), so a +// default round trip drops those; use roundTripFull to exercise their recovery. +function roundTrip(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources(entries, entryLinks); +} + +// A round trip through a `--emit-expressions` push: the catalog then holds the +// per-field `semantics` block and the metric expression, so field/metric +// expressions and DIMENSION roles round-trip too. +function roundTripFull(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries, entryLinks} = + generateCatalogResources(model, {...OPTS, emitExpressions: true}); + return modelsFromCatalogResources(entries, entryLinks); +} + + +describe('emitter -> reader round trip (lossless slice)', () => { + // A model using only round-trippable content: no keys/ai_context/labels/ + // relationships (all dropped by the write), and datatypes that invert + // cleanly. Its fields and metric carry expressions and a DIMENSION role, so + // it round-trips losslessly only through a `--emit-expressions` push + // (roundTripFull); a default push omits the per-field `semantics` block. + const source: SemanticModel = { + name: 'sales', + description: 'the sales model', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], // keys are not persisted; keep empty so the round trip matches + fields: [ + {name: 'o_orderkey', expression: 'orders.o_orderkey', type: 'Integer'}, + { + name: 'o_orderdate', + expression: 'orders.o_orderdate', + type: 'Date', + dimension: {}, + description: 'order date', + }, + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + type: 'Decimal', + }], + }; + + test('reconstructs an IR equal to the source', () => { + const {models} = roundTripFull(source); + expect(models).toHaveLength(1); + expect(models[0]).toEqual(source); + }); +}); + + +describe('relationship recovery (schema-join links -> IR)', () => { + // orders.o_custkey (the foreign-key side) references customer.c_custkey. + const twoEntities: Entity[] = [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ]; + + test( + 'a 1:N relationship recovers its endpoints, direction, and columns', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: + 'places', // already link-slug-safe, so it round-trips exactly + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const rels = roundTrip(model).models[0].relationships; + expect(rels).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'a relationship name comes back normalized (lowercased/hyphenated)', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'Places_Order', // mixed case + underscore -> normalized on + // read + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('places-order'); + }); + + test('a many-to-many (association) relationship is not recovered', () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'enrolls', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + association: { + dataSource: 'p.d.junction', + keys: ['id'], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + }, + }], + metrics: [], + }; + // The emitter never publishes M:N, so no schema-join link exists to read. + expect(roundTrip(model).models[0].relationships).toEqual([]); + }); + + test( + 'endpoints resolve from the link references, not the shared table name', + () => { + // Two entities over the SAME table: a data-source index would collapse + // them (last write wins), so both endpoints must come from the link's + // entryReferences instead. Direction can't be told from the shared + // table, so the reader keeps the reference order and warns. + const sameTable: Entity[] = [ + { + name: 'parent_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'id', expression: 'parent_order.id'}], + }, + { + name: 'child_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'parent_id', expression: 'child_order.parent_id'}], + }, + ]; + const model: SemanticModel = { + name: 'sales', + entities: sameTable, + relationships: [{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }], + metrics: [], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].relationships).toEqual([{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }]); + expect(warnings.some(w => /join direction is ambiguous/i.test(w))) + .toBe(true); + }); + + test( + 'endpoints resolve when link references carry an un-normalized project ' + + 'number', + () => { + // The live path is asymmetric: lookupEntry normalizes entry names to + // project IDs, but lookupEntryLinks returns references still carrying + // the numeric project. Matching on the (stable) entry id keeps the + // relationship resolvable rather than silently dropping it. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const numeric = entryLinks.map( + l => ({ + ...l, + entryReferences: + (l.entryReferences ?? + []).map(r => ({ + ...r, + name: r.name.replace( + 'projects/dest/', 'projects/000000000000/'), + })), + })); + const {models} = modelsFromCatalogResources(entries, numeric); + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'the model prefix is stripped across tricky model names ' + + '(linkNamePrefix tracks the emitter slug)', + () => { + // Pins the read-side prefix reproduction to the emitter's linkSlug: if + // it drifts, the model prefix would survive and the recovered name + // would not equal the bare, normalized relationship name. + for (const modelName of ['Sales', 'Sales Analytics', '123 Sales!']) { + const model: SemanticModel = { + name: modelName, + entities: twoEntities, + relationships: [{ + name: 'rel_one', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('rel-one'); + } + }); + + test('an unnamed link returned from both endpoints is deduped', () => { + // Defense in depth: if the catalog ever returns a nameless schema-join + // link, the per-endpoint fan-out yields it twice; dedup falls back to the + // endpoint pair so it becomes one relationship, not two. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {models} = modelsFromCatalogResources( + entries, [...nameless, ...nameless.map(l => ({...l}))]); + expect(models[0].relationships).toHaveLength(1); + }); +}); + + +describe( + 'deployment-target recovery (semantic-model aspect -> custom_extensions)', + () => { + test('the GOOGLE deployment targets ride back verbatim', () => { + const uri = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g'; + const data = JSON.stringify({deploymentTargets: [uri]}); + const model: SemanticModel = { + name: 'sales', + customExtensions: [{vendorName: 'GOOGLE', data}], + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toEqual([ + {vendorName: 'GOOGLE', data} + ]); + }); + + test( + 'a model with no deployment targets recovers no custom_extensions', + () => { + const model: SemanticModel = { + name: 'sales', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toBeUndefined(); + }); + }); + + +describe('dataType inverse (schema aspect -> IR type)', () => { + // Emit a one-field model of each IR type, read it back, and check the field's + // reconstructed type. String and Opaque both emit dataType STRING; String + // (indistinguishable from an un-typed field) reads back as undefined, while + // Opaque is disambiguated by metadataType OTHER. + const cases: [Metric['type']|undefined, Metric['type']|undefined][] = [ + ['Integer', 'Integer'], + ['Decimal', 'Decimal'], + ['Float', 'Float'], + ['Boolean', 'Boolean'], + ['Date', 'Date'], + ['Time', 'Time'], + ['DateTime', 'DateTime'], + ['DateTimeTz', 'DateTimeTz'], + ['Opaque', 'Opaque'], + ['String', undefined], // collapses to un-typed + [undefined, undefined], // un-typed stays un-typed + ]; + + for (const [type, expected] of cases) { + test(`${type ?? 'un-typed'} -> ${expected ?? 'un-typed'}`, () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f', ...(type ? {type} : {})}], + }], + relationships: [], + metrics: [], + }; + const back = roundTrip(model).models[0].entities[0].fields[0]; + expect(back.type).toBe(expected as any); + }); + } +}); + + +describe('field mapping details', () => { + function readWith( + rt: (m: SemanticModel) => {models: SemanticModel[]}, + field: Entity['fields'][number]) { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: [field]}], + relationships: [], + metrics: [], + }; + return rt(model).models[0].entities[0].fields[0]; + } + // Default push (no per-field `semantics`) vs a `--emit-expressions` push. + const readField = (f: Entity['fields'][number]) => readWith(roundTrip, f); + const readFieldFull = (f: Entity['fields'][number]) => + readWith(roundTripFull, f); + + test('a DIMENSION role reads back as a dimension marker', () => { + // The role lives in the gated `semantics` block, so it survives only a + // `--emit-expressions` push. + const back = readFieldFull({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toEqual({}); + }); + + test('a non-dimension field has no dimension marker', () => { + const back = readFieldFull({name: 'f', expression: 'e.f'}); + expect(back.dimension).toBeUndefined(); + }); + + test( + 'a default push drops the DIMENSION role with the semantics block', + () => { + const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toBeUndefined(); + expect(back.expression).toBeUndefined(); + }); + + test( + 'an imported expression is not persisted to or recovered from the catalog', + () => { + const back = readFieldFull({ + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }); + expect(back.expression).toBe('e.amt'); + // The emitter no longer writes importedExpression/importedDialect, so + // neither survives the round trip. + expect(back.importedExpression).toBeUndefined(); + expect(back.importedDialect).toBeUndefined(); + }); + + test('a schema field missing its name is skipped with a warning', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f'}], + }], + relationships: [], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + // Inject a nameless field record into the entity's schema aspect, as a + // malformed catalog could return; the reader must drop it, not emit a field + // with an undefined name. + for (const entry of entries) { + for (const aspect of Object.values(entry.aspects ?? {})) { + const fields = (aspect as {data?: {fields?: unknown[]}}).data?.fields; + if (Array.isArray(fields)) fields.push({dataType: 'INT64'}); + } + } + const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); + expect(models[0].entities[0].fields.map(f => f.name)).toEqual(['f']); + expect( + warnings.some(w => /missing its name; the field is skipped/i.test(w))) + .toBe(true); + }); +}); + + +describe('data source resource-path parsing', () => { + function readDataSource(dataSource: string): string { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource, keys: [], fields: []}], + relationships: [], + metrics: [], + }; + return roundTrip(model).models[0].entities[0].dataSource; + } + + test( + 'a three-part BigQuery reference round-trips through the resource URI', + () => { + expect(readDataSource('proj.ds.tbl')).toBe('proj.ds.tbl'); + }); + + test('a verbatim query source is preserved unchanged', () => { + const query = 'SELECT * FROM t'; + expect(readDataSource(query)).toBe(query); + }); +}); + + +describe('metric attach entity is re-derived from the expression', () => { + test( + 'a single-entity metric attaches; a cross-entity metric does not', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + metrics: [ + {name: 'revenue', expression: 'SUM(orders.amt)', entity: 'orders'}, + { + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)' + }, + ], + }; + // Full push: the expression is in the catalog, so the entity is + // re-derived from it (single entity -> attach; two entities -> none). + const {models} = roundTripFull(model); + const byName = new Map(models[0].metrics.map(m => [m.name, m])); + expect(byName.get('revenue')!.entity).toBe('orders'); + expect(byName.get('mix')!.entity).toBeUndefined(); + }); + + test( + 'a cross-entity metric falls back to the persisted attach entity', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + // Two entities in the expression, so it cannot be re-derived; the + // authored attach entity must ride back on the persisted aspect. + metrics: [{ + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)', + entity: 'orders', + }], + }; + const {models} = roundTrip(model); + expect(models[0].metrics[0].entity).toBe('orders'); + }); +}); + + +describe('anchor / parent grouping', () => { + test('no semantic-model entry yields no models and a warning', () => { + const {models, warnings} = modelsFromCatalogResources([]); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic-model entry/i.test(w))).toBe(true); + }); + + test('two models keep their own children by parentEntry', () => { + const a = generateCatalogResources( + { + name: 'a', + entities: [{name: 'ea', dataSource: 'p.d.a', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const b = generateCatalogResources( + { + name: 'b', + entities: [{name: 'eb', dataSource: 'p.d.b', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const {models} = modelsFromCatalogResources([...a.entries, ...b.entries]); + const byName = new Map(models.map(m => [m.name, m])); + expect(byName.get('a')!.entities.map(e => e.name)).toEqual(['ea']); + expect(byName.get('b')!.entities.map(e => e.name)).toEqual(['eb']); + }); +}); + + +describe('metric expression referencing no known entity', () => { + test( + 'warns that the metric may be unplaceable and leaves it unattached', + () => { + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], + relationships: [], + // References `widgets`, which is not an entity of this model. + metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], + }; + // The unplaceable warning fires from the expression, so it needs a push + // that wrote one (a default push omits it, leaving nothing to check). + const {models, warnings} = roundTripFull(model); + expect(models[0].metrics[0].entity).toBeUndefined(); + expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) + .toBe(true); + }); +}); + + +// -- Golden pull: the whole KC entries -> IR -> OSI YAML output. -- +// +// The round trip above proves the reader inverts the emitter in memory; this +// pins the reviewable artifact. For each corpus fixture it reads the committed +// emitter golden (`.knowledge_catalog.golden.json` -- the exact +// entries and entry links a push produced) back through the reader and +// serializes the reconstructed IR to `.pull.golden.yaml`. Open that +// next to the fixture's `.osi.golden.yaml` to see, as whole files, what a +// Knowledge Catalog round trip preserves (including 1:1 / 1:N relationships and +// deployment targets) and what it drops (keys, ai_context, labels, vendor SQL, +// M:N relationships). +// +// Regenerate after an intentional reader/serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts +describe( + 'golden pull: each corpus KC golden reconstructs to its exact YAML', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + const kcGoldenPath = (fixture: string) => path.join( + FIXTURES, + fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + const pullGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const {models, warnings} = + modelsFromCatalogResources(kc.entries, kc.entryLinks ?? []); + // Reader warnings ride along as YAML comments so the golden shows + // the full outcome, not just the recovered document. + const header = warnings.length ? + warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : + '# (no warnings)\n'; + const actual = + header + models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = pullGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename( + golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); + + +// -- Symmetry: emit -> read drops ONLY the documented set. -- +// +// The golden pull above lets a human eyeball what a Knowledge Catalog round +// trip loses; this asserts it mechanically. For each corpus fixture it loads +// the authored IR, runs a full emit -> read round trip, and asserts the result +// equals the authored IR reduced to the "KC floor" -- the model with exactly +// the documented losses and normalizations applied (`stripToKcFloor`). Applying +// that same reduction to BOTH sides makes `toEqual` flag only UNDOCUMENTED +// divergence, so a new emitter/reader regression (a dropped column, a lost +// description, an un-stripped M:N edge) fails here even though every individual +// loss is already pinned by a targeted test above. +describe( + 'symmetry: an emit -> read round trip drops only documented fields', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the OSI / KC / pull goldens. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + for (const authored of loadModels(text, LOAD).models) { + const {models, warnings} = roundTrip(authored); + expect(models).toHaveLength(1); + expect(stripToKcFloor(models[0])).toEqual(stripToKcFloor(authored)); + // A clean corpus model round-trips without reader warnings. + expect(warnings).toEqual([]); + } + }); + } + }); + + +// Reduces a model to the "KC floor": the most an emit -> read round trip can +// preserve, i.e. the authored model with exactly the documented Knowledge +// Catalog losses and normalizations applied. Idempotent (already-floored input +// is unchanged), so it can be applied to both sides of a round trip. Each +// transform mirrors one emitter/reader behavior pinned by a targeted test +// above. +function stripToKcFloor(model: SemanticModel): SemanticModel { + const m = structuredClone(model); + + // Model ai_context is not persisted; only a GOOGLE deployment-targets block + // rides back, re-serialized to the reader's canonical JSON form. + delete m.aiContext; + const ext = canonicalGoogleExt(m.customExtensions); + if (ext) { + m.customExtensions = ext; + } else { + delete m.customExtensions; + } + + for (const e of m.entities) { + e.keys = []; // keys / unique keys are never persisted + delete e.uniqueKeys; + delete e.aiContext; + delete e.customExtensions; + for (const f of e.fields) { + delete f.label; // display label is not persisted + delete f.aiContext; + delete f.importedExpression; // vendor SQL is not persisted + delete f.importedDialect; + delete f.customExtensions; + // A default push omits the per-field `semantics` block, so the field + // expression and the DIMENSION role are not persisted (they ride back + // only through a `--emit-expressions` push). + delete f.expression; + delete f.dimension; + // String is indistinguishable from an un-typed field on read. + if (f.type === 'String') delete f.type; + } + } + + for (const metric of m.metrics) { + delete metric.aiContext; + delete metric.importedExpression; + delete metric.importedDialect; + delete metric.customExtensions; + delete metric.expression; // omitted by a default push, like field ones + // The emitter writes a required dataType, defaulting typeless -> NUMERIC, + // which reads back as Decimal; String collapses to un-typed like fields. + if (metric.type === undefined) { + metric.type = 'Decimal'; + } else if (metric.type === 'String') { + delete metric.type; + } + } + + // M:N (association) edges are never published; a 1:1 / 1:N name comes back + // normalized via the emitter's slug (its exact form is pinned above). + m.relationships = m.relationships.filter(r => !r.association).map(r => { + const rel = structuredClone(r); + rel.name = linkNamePrefix(rel.name); + delete rel.aiContext; + delete rel.customExtensions; + return rel; + }); + + return m; +} + + +// The inverse-canonical of the emitter's deployment-target persistence, +// matching the reader's readDeploymentTargets: keep only GOOGLE +// deploymentTargets and re-serialize them so an authored block and its +// round-tripped form (which the reader always JSON.stringifies afresh) compare +// equal. Returns undefined when the model declares none, exactly as the reader +// omits custom_extensions then. +function canonicalGoogleExt(exts: CustomExtension[]|undefined): + CustomExtension[]|undefined { + const targets: string[] = []; + for (const ext of exts ?? []) { + if (ext.vendorName !== 'GOOGLE') continue; + let parsed: any; + try { + parsed = JSON.parse(ext.data); + } catch { + continue; + } + for (const t of parsed?.deploymentTargets ?? []) { + if (typeof t === 'string' && t) targets.push(t); + } + } + return targets.length ? [{ + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }] : + undefined; +} diff --git a/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts new file mode 100644 index 00000000..82943898 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts @@ -0,0 +1,334 @@ +// Behavior specification for the OSI converter's serialize direction +// (serializeModel in src/libts/semantic/osi_converter.ts). +// +// The serializer is the inverse of loader.ts: IR -> open-format YAML. The +// strongest guarantee is a round trip through the loader -- load a fixture to +// the IR, serialize it, load the serialized text again, and assert the two IRs +// are identical. That pins IR-level fidelity across every feature the loader +// produces (datasets, fields, datatypes, dimensions, labels, ai_context, +// custom_extensions, relationships, metrics, imported expressions) without +// hard-coding YAML text. Targeted structural tests cover the mapping details a +// round trip cannot isolate. + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as yaml from 'yaml'; + +import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; +import {loadModels} from '../../../src/libts/semantic/loader'; +import {modelDocument, serializeModel} from '../../../src/libts/semantic/osi_converter'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +function loadFixture(name: string): SemanticModel[] { + const text = fs.readFileSync(path.join(FIXTURES, name), 'utf8'); + return loadModels(text).models; +} + + +describe('loader <-> serialize round trip is IR-stable', () => { + // Each fixture exercises a different slice of the format: relationships + + // ai_context + synonyms + label + dimension; the full tpc-ds corpus with + // custom_extensions + unique_keys; explicit datatypes; and imported + // (vendor-dialect) expressions. + const fixtures = [ + 'star_orders_customer.yaml', + 'tpcds_retail.yaml', + 'sales_google_ext.yaml', + 'vendor_dialects.yaml', + 'lineitem_databricks_ext.yaml', + 'sales_bq_graph_target.yaml', + ]; + + for (const fixture of fixtures) { + test(`${fixture} survives IR -> YAML -> IR unchanged`, () => { + const original = loadFixture(fixture); + expect(original.length).toBeGreaterThan(0); + + for (const model of original) { + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models; + expect(reloaded).toHaveLength(1); + // IR-level equality: every field the loader keeps must match exactly. + expect(reloaded[0]).toEqual(model); + } + }); + } +}); + + +describe('serialized document structure', () => { + const model = loadFixture('star_orders_customer.yaml')[0]; + const doc = modelDocument(model) as any; + const sm = doc.semantic_model[0]; + + test('emits the supported version and a single model', () => { + expect(doc.version).toBe('0.2.0.dev0'); + expect(doc.semantic_model).toHaveLength(1); + expect(sm.name).toBe(model.name); + }); + + test('a dataset source is the opaque dataSource string, verbatim', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.source).toBe(entity.dataSource); + expect(typeof orders.source).toBe('string'); + }); + + test('primary_key mirrors the entity keys', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.primary_key).toEqual(entity.keys); + }); + + test('ai_context is emitted structurally (synonyms round-trip)', () => { + // The fixture annotates a field (o_orderdate) with synonyms; find it and + // assert the structured ai_context is emitted under that field. + const entity = model.entities.find( + e => e.fields.some(f => f.aiContext?.synonyms?.length))!; + const field = entity.fields.find(f => f.aiContext?.synonyms?.length)!; + const dsDoc = sm.datasets.find((d: any) => d.name === entity.name); + const fieldDoc = dsDoc.fields.find((f: any) => f.name === field.name); + expect(fieldDoc.ai_context.synonyms).toEqual(field.aiContext!.synonyms); + }); + + test('a relationship maps to from/to + positional columns', () => { + expect(sm.relationships.length).toBeGreaterThan(0); + const rel = model.relationships[0]; + const relDoc = sm.relationships[0]; + expect(relDoc.from).toBe(rel.source.entity); + expect(relDoc.to).toBe(rel.destination.entity); + expect(relDoc.from_columns).toEqual(rel.source.columns); + expect(relDoc.to_columns).toEqual(rel.destination.columns); + }); +}); + + +describe('expression + datatype + dimension mapping', () => { + test('an explicit datatype round-trips as `datatype`', () => { + const model = loadFixture('sales_google_ext.yaml')[0]; + const typed = model.entities.flatMap(e => e.fields).find(f => f.type); + expect(typed).toBeDefined(); + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models[0]; + const back = reloaded.entities.flatMap(e => e.fields) + .find(f => f.name === typed!.name)!; + expect(back.type).toBe(typed!.type); + }); + + test('a bare dimension marker survives as `dimension: {}`', () => { + const field: + Field = {name: 'ship_date', expression: 'e.ship_date', dimension: {}}; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const fieldDoc = doc.semantic_model[0].datasets[0].fields[0]; + expect(fieldDoc.dimension).toEqual({}); + // And it reloads back to a dimension field. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.entities[0].fields[0].dimension).toEqual({}); + }); + + test('an imported vendor expression is emitted under its own dialect', () => { + const field: Field = { + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const dialects = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects; + const labels = dialects.map((d: any) => d.dialect); + expect(labels).toContain('SNOWFLAKE'); + // The canonical form is labeled BIGQUERY so the loader re-picks it exactly. + expect(labels).toContain('BIGQUERY'); + }); + + test('a metric does not emit its derived attach entity', () => { + const metric: Metric = { + name: 'total', + expression: 'SUM(orders.amt)', + entity: 'orders', + }; + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{name: 'amt', expression: 'orders.amt'}], + }], + relationships: [], + metrics: [metric], + }; + const metricDoc = + (modelDocument(model) as any).semantic_model[0].metrics[0]; + expect(metricDoc).not.toHaveProperty('entity'); + // The loader re-derives it on reload. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.metrics[0].entity).toBe('orders'); + }); +}); + + +describe('lossy edges are flagged', () => { + test( + 'an association relationship warns and drops the junction detail', () => { + const rel: Relationship = { + name: 'enrollment', + source: {entity: 'student', columns: ['id']}, + destination: {entity: 'course', columns: ['id']}, + association: { + dataSource: 'p.d.enrollment', + keys: ['student_id', 'course_id'], + sourceColumns: ['student_id'], + destinationColumns: ['course_id'], + }, + }; + const model: SemanticModel = { + name: 'school', + entities: [ + { + name: 'student', + dataSource: 'p.d.student', + keys: ['id'], + fields: [] + }, + { + name: 'course', + dataSource: 'p.d.course', + keys: ['id'], + fields: [] + }, + ], + relationships: [rel], + metrics: [], + }; + const {yaml: text, warnings} = serializeModel(model); + expect(warnings.some(w => /association/i.test(w))).toBe(true); + // The direct-FK view is still emitted (from/to + columns), so it + // reloads. + const relDoc = yaml.parse(text).semantic_model[0].relationships[0]; + expect(relDoc.from).toBe('student'); + expect(relDoc.to).toBe('course'); + expect(relDoc.from_columns).toEqual(['id']); + }); +}); + + +describe('serialize flags loader-invalid reconstructions', () => { + test('a model with no entities warns that datasets is required', () => { + const model: SemanticModel = { + name: 'empty', + entities: [], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /no datasets/i.test(w))).toBe(true); + }); + + test('a field with no expression warns that one is required', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'orphan'}], + }], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /field 'orphan'.*no expression/i.test(w))) + .toBe(true); + }); + + test( + 'an imported dialect colliding with the canonical label is relabeled', + () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{ + name: 'amt', + expression: 'orders.amt', + importedExpression: 'orders.AMT', + importedDialect: 'BIGQUERY', + }], + }], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const labels: string[] = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects.map( + (d: any) => d.dialect); + // No duplicate dialect label: the imported form is relabeled off + // BIGQUERY so the loader does not pick between two BIGQUERY entries. + expect(new Set(labels).size).toBe(labels.length); + expect(labels).toContain('BIGQUERY'); + }); +}); + + +// -- Golden corpus: the whole IR -> OSI YAML output, reviewable as a file. -- +// +// The round-trip tests above prove IR-level stability but never pin the exact +// YAML text. These goldens capture the full serialized document for a small +// corpus so a reviewer can open a `.yaml` next to its `.osi.golden.yaml` and see +// exactly what the serializer emits. The same corpus backs kc_converter's +// `.pull.golden.yaml`, so diffing the two shows what a Knowledge Catalog round +// trip loses relative to the authored source. +// +// Regenerate after an intentional serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/osi_converter.test.ts +describe('golden OSI document: each corpus fixture serializes to its exact YAML', + () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the KC e2e/pull goldens, so the OSI golden + // and the pull golden are directly comparable. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + const osiGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.osi.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + const models = loadModels(text, LOAD).models; + const actual = models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = osiGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts new file mode 100644 index 00000000..83f5aee0 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -0,0 +1,352 @@ +// Tests for the semantic-model Knowledge Catalog pull leg +// (pullKnowledgeCatalog in src/libts/semantic/pull_kc.ts). +// +// pullKnowledgeCatalog is the orchestration around the pure reader: enumerate +// the entry group, hydrate each semantic entry's aspect data, and reconstruct +// the IR. The catalog client is stubbed so no network call is made. The entries +// the fake serves are produced by the real emitter, so this exercises the true +// list -> hydrate -> read path end to end (an entity is re-fetched with BOTH +// its semantic-entity and schema aspects, and a second pass fetches each +// entity's schema-join links). The reader's own mapping is covered in +// kc_converter.test.ts; the focus here is the fetch SEQUENCE: aspect hydration, +// the relationship-link fetch (per-entry, deduped across endpoints), the +// --model filter, skipped entries, and ignoring foreign entries. + +import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; + +import {ApiResult} from '../../../src/libts/gcp/api'; +import {CatalogClient, Entry, EntryLink} from '../../../src/libts/gcp/dataplex'; +import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +const SALES: SemanticModel = { + name: 'sales', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [ + {name: 'o_totalprice', expression: 'orders.o_totalprice', type: 'Decimal'} + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + // Metrics carry a datatype through KC (semantic-metric.dataType is + // required); Decimal -> NUMERIC on emit, NUMERIC -> Decimal on read. + type: 'Decimal' + }], +}; + +// The entries the emitter would have written for a model. Emitted with +// expressions on (a `--emit-expressions` push) so a model that carries field / +// metric expressions reconstructs exactly; the default push omits them (the +// converter tests in kc_converter.test.ts pin that gating). +function entriesFor(model: SemanticModel): Entry[] { + return generateCatalogResources(model, {...OPTS, emitExpressions: true}) + .entries; +} + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +// Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, +// or 404s an unknown name). `lookupFail` forces a failure for one entry name. +// `links` are served by lookupEntryLinks per referenced entry (so an undirected +// link comes back once from each endpoint, exercising the puller's dedup); +// `linkFail` forces a link-lookup failure for one entry name. +function stubClient( + listed: Entry[], served: Entry[], lookupFail?: string, + links: EntryLink[] = [], + linkFail?: string): {list: any; lookup: any; lookupLinks: any} { + const byName = new Map(served.map(e => [e.name, e])); + const list = spyOn(CatalogClient.prototype, 'listEntries') + .mockImplementation(async function*() { + for (const e of listed) yield e; + } as any); + const lookup = spyOn(CatalogClient.prototype, 'lookupEntry') + .mockImplementation(async (_p, _l, name) => { + if (name === lookupFail) return err(500, 'boom'); + const e = byName.get(name); + return e ? ok(e) : err(404, 'not found'); + }); + const lookupLinks = + spyOn(CatalogClient.prototype, 'lookupEntryLinks') + .mockImplementation(async (_p: any, _l: any, opts: any) => { + if (opts.entry === linkFail) return err(500, 'boom'); + const forEntry = links.filter( + l => + (l.entryReferences ?? []).some(r => r.name === opts.entry)); + return ok(forEntry); + }); + return {list, lookup, lookupLinks}; +} + +afterEach(() => { + mock.restore(); +}); + + +describe('pullKnowledgeCatalog: happy path', () => { + test('reconstructs the model from listed + hydrated entries', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models).toHaveLength(1); + expect(models[0]).toEqual(SALES); + expect(warnings).toHaveLength(0); + // Every listed semantic entry (model + entity + metric) was hydrated. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + 'an entity is hydrated with BOTH its semantic-entity and schema aspects', + async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + await pullKnowledgeCatalog(cat, OPTS); + + // Find the lookup call for the entity entry and inspect its requested + // aspect types (the 4th arg). + const entityEntry = + entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + const call = + lookup.mock.calls.find((c: any[]) => c[2] === entityEntry.name)!; + const aspectTypes: string[] = call[3]; + expect( + aspectTypes.some(t => t.endsWith('/aspectTypes/semantic-entity'))) + .toBe(true); + expect(aspectTypes.some(t => t.endsWith('/aspectTypes/schema'))) + .toBe(true); + }); +}); + + +describe('pullKnowledgeCatalog: relationship links', () => { + // A 1:N model: orders.o_custkey (the FK side) references customer.c_custkey. + const SALES_REL: SemanticModel = { + name: 'sales', + entities: [ + { + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'demo.sales.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ], + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + + test( + 'a schema-join link reconstructs into a relationship exactly once', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const {lookupLinks} = + stubClient(entries, entries, undefined, entryLinks); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + expect(warnings).toHaveLength(0); + // Links are fetched per entity (2 entities), and the single undirected + // link -- returned from both endpoints -- is deduped to one edge. + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + + test( + 'an unnamed link returned from both endpoints is deduped to one edge', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + // A nameless link can't dedup by name; the puller must fall back to the + // endpoint pair, or the per-endpoint fan-out would yield it twice. + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {lookupLinks} = stubClient(entries, entries, undefined, nameless); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + + // Deduped to a single edge (a nameless link has no id, so the name + // itself can't be recovered -- but it must not be counted twice). + expect(models[0].relationships).toHaveLength(1); + expect(models[0].relationships[0].source.entity).toBe('orders'); + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + + test( + 'a failed link lookup on one endpoint still recovers the edge from the ' + + 'other, and warns', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const ordersEntry = + entries.find(e => (e.entrySource?.displayName) === 'orders')!; + // Fail the link lookup for the orders entity; the link still comes back + // from the customer endpoint. + stubClient(entries, entries, undefined, entryLinks, ordersEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships.map(r => r.name)).toEqual(['places']); + expect(warnings.some( + w => /failed to fetch entry links/i.test(w) && + w.includes(ordersEntry.name))) + .toBe(true); + }); +}); + + +describe('pullKnowledgeCatalog: filtering and robustness', () => { + test('--model keeps only the named model', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + }); + + test('--model with no match returns nothing and warns', async () => { + const entries = entriesFor(SALES); + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); + + test( + 'a non-semantic entry in the group is ignored, not fetched', async () => { + const entries = entriesFor(SALES); + const foreign: Entry = { + name: `projects/dest/locations/us/entryGroups/eg/entries/foreign`, + entryType: 'projects/x/locations/us/entryTypes/some-other-type', + }; + const {lookup} = stubClient([...entries, foreign], entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + expect(models).toHaveLength(1); + // The foreign entry is never hydrated (only the 3 semantic entries + // are). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('a failed hydration is skipped with a warning', async () => { + const entries = entriesFor(SALES); + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + stubClient(entries, entries, metricEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + // The model + entity still reconstruct; only the metric is dropped. + expect(models).toHaveLength(1); + expect(models[0].metrics).toHaveLength(0); + expect(warnings.some( + w => /failed to fetch/i.test(w) && w.includes(metricEntry.name))) + .toBe(true); + }); + + test('--model hydrates only the target model\'s entries', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + // SALES has 3 entries (model + entity + metric); inventory's are never + // fetched -- the flag scopes hydration, not just the final result. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + '--model keeps a child whose parentEntry does not resolve (sole-anchor ' + + 'fallback)', + async () => { + const entries = entriesFor(SALES); + // Simulate a project-id normalization mismatch: the metric points at an + // anchor name that differs from the emitted one. With a single model in + // the group a full pull still attaches it via the reader's sole-anchor + // fallback, so a scoped pull must keep it too (else --model silently + // drops a child a full pull returns). + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + metricEntry.parentEntry = metricEntry.parentEntry!.replace( + 'projects/dest/', 'projects/12345/'); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + + expect(models).toHaveLength(1); + expect(models[0].metrics.map(m => m.name)).toEqual(['total_revenue']); + // The metric was hydrated despite the parent mismatch (3 entries). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('--model with no match fetches nothing and warns', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(lookup).not.toHaveBeenCalled(); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts new file mode 100644 index 00000000..ac4c7bc0 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts @@ -0,0 +1,75 @@ +// Tests for the SemanticModel layout's pull write-path +// (src/libts/layouts/semantic-model.ts): modelPath / hasModel / +// writeModelDocument. These are the sink `pull` writes reconstructed models to; +// the push-side discovery (modelDocuments) is exercised via the deploy tests. + +import {afterEach, beforeEach, describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import {SemanticModelLayout} from '../../../src/libts/layouts/semantic-model'; + +let root: string; +let catalogPath: string; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'kcmd-layout-')); + catalogPath = path.join(root, 'catalog'); +}); + +afterEach(() => { + fs.rmSync(root, {recursive: true, force: true}); +}); + +async function layout(entryGroup?: string): Promise { + const l = new SemanticModelLayout(catalogPath, entryGroup); + await l.init(); + return l; +} + + +describe('SemanticModelLayout write path', () => { + test('modelPath maps to EntryGroups//.yaml', async () => { + const l = await layout('eg'); + expect(l.modelPath('sales')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'sales.yaml')); + }); + + test('modelPath sanitizes path separators in the model name', async () => { + const l = await layout('eg'); + expect(l.modelPath('a/b')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'a_b.yaml')); + }); + + test('modelPath throws without an entry group', async () => { + const l = await layout(undefined); + expect(() => l.modelPath('sales')).toThrow(/entry group/i); + }); + + test( + 'writeModelDocument creates the file, dirs, and indexes it', async () => { + const l = await layout('eg'); + expect(l.hasModel('sales')).toBe(false); + + l.writeModelDocument('sales', 'version: x\n'); + + const p = l.modelPath('sales'); + expect(fs.existsSync(p)).toBe(true); + expect(fs.readFileSync(p, 'utf8')).toBe('version: x\n'); + expect(l.hasModel('sales')).toBe(true); + // Indexed, so a subsequent read surfaces it as a model document. + expect(l.modelDocuments()).toEqual([ + {name: 'sales', text: 'version: x\n'} + ]); + }); + + test( + 'writeModelDocument overwrites an existing document (last-write-wins)', + async () => { + const l = await layout('eg'); + l.writeModelDocument('sales', 'first\n'); + l.writeModelDocument('sales', 'second\n'); + expect(fs.readFileSync(l.modelPath('sales'), 'utf8')).toBe('second\n'); + }); +});