From d8a4c633100bd39045a04afe21cd58278581bdb3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 04:44:39 +0000 Subject: [PATCH 01/12] mdcode: add semantic-model pull from Knowledge Catalog Add the inverse of the KC emitter: read semantic-model / -entity / -metric entries and their aspects back into the IR, serialize to YAML, and wire a 'pull' command (with --dry-run and --model) for the semantic-model scope. Re-stacked onto the KC-push follow-ups: the emitter no longer writes importedExpression, so the reader no longer recovers it; idOf is shared from knowledge_catalog.ts; and push entry/link writes use the same bounded mapConcurrent pool as pull hydration. --- .../src/libts/layouts/semantic-model.ts | 30 ++ .../semantic/deploy_knowledge_catalog.ts | 199 +++++++++++- .../src/libts/semantic/knowledge_catalog.ts | 263 +++++++++++++++- .../mdcode/src/libts/semantic/serialize.ts | 256 ++++++++++++++++ toolbox/mdcode/src/tool/commands.ts | 90 +++++- toolbox/mdcode/src/tool/main.ts | 6 +- .../deploy_knowledge_catalog.pull.test.ts | 244 +++++++++++++++ .../semantic/knowledge_catalog.read.test.ts | 258 ++++++++++++++++ .../semantic/semantic_model_layout.test.ts | 75 +++++ .../tests/libts/semantic/serialize.test.ts | 290 ++++++++++++++++++ 10 files changed, 1695 insertions(+), 16 deletions(-) create mode 100644 toolbox/mdcode/src/libts/semantic/serialize.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/serialize.test.ts diff --git a/toolbox/mdcode/src/libts/layouts/semantic-model.ts b/toolbox/mdcode/src/libts/layouts/semantic-model.ts index 214fe930..42730af7 100644 --- a/toolbox/mdcode/src/libts/layouts/semantic-model.ts +++ b/toolbox/mdcode/src/libts/layouts/semantic-model.ts @@ -91,6 +91,36 @@ export class SemanticModelLayout implements CatalogLayout { return docs; } + // True when a model document with this handle already exists on disk. `pull` + // uses it to report which files it would overwrite vs. create. + hasModel(name: string): boolean { + return fs.existsSync(this.modelPath(name)); + } + + // The absolute path a model document with this handle maps to: + // `/EntryGroups//.yaml`. Path separators in the + // model name are replaced so a name still yields a single flat file. Requires + // the layout to be scoped to an entry group (the semantic-model source always + // is). + modelPath(name: string): string { + if (!this._entryGroup) { + throw new Error( + 'SemanticModel layout has no entry group; cannot resolve a model path.'); + } + const file = `${name.replace(/[/\\]/g, '_')}.yaml`; + return path.join(this._catalogPath, 'EntryGroups', this._entryGroup, file); + } + + // Writes a model's serialized document to its path, creating the EntryGroup + // directory if needed, and indexes it so a later modelDocuments() sees it. + // This is the sink `pull` writes reconstructed models to. + writeModelDocument(name: string, text: string): void { + const localPath = this.modelPath(name); + fs.mkdirSync(path.dirname(localPath), {recursive: true}); + fs.writeFileSync(localPath, text); + this._index.set(name, localPath); + } + // The Knowledge Catalog entry-level members are not applicable to this // push-only layout; the model is authored as a single Ossie document, not as // per-entry Knowledge Catalog files. These are wired when KC-resource emit diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 5676ec86..0bf69527 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -38,7 +38,8 @@ import {ApiResult} from '../gcp/api'; import * as context from '../gcp/context'; import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; -import {generateCatalogResources, KcResources} from './knowledge_catalog'; +import {SemanticModel} from './ir'; +import {generateCatalogResources, idOf, KcResources, modelsFromCatalogResources} from './knowledge_catalog'; import {LoadedModel} from './loader'; @@ -120,6 +121,10 @@ interface Counts { type EmittedModel = {model: string; resources: KcResources}; +// Upper bound on concurrent entry / entry-link writes within one model's wave +// (mirrors HYDRATE_CONCURRENCY on the pull side). +const WRITE_CONCURRENCY = 8; + // entries.create propagation retry: a just-created entry group can briefly 404. const ENTRY_CREATE_TRIES = 3; const ENTRY_CREATE_RETRY_MS = 3000; @@ -566,7 +571,8 @@ async function createEntries( const isMetric = (e: Entry) => (e.entryType ?? '').endsWith('/semantic-metric'); for (const wave of [children.filter(e => !isMetric(e)), children.filter(isMetric)]) { - const res = await Promise.all(wave.map(e => writeEntry(cat, opts, e))); + const res = await mapConcurrent( + wave, WRITE_CONCURRENCY, e => writeEntry(cat, opts, e)); const firstErr = res.find(r => r.error); if (firstErr) return {created, updated, error: firstErr.error}; for (const r of res) { @@ -593,7 +599,8 @@ async function createEntryLinks( cat: CatalogClient, opts: KcDeployOptions, links: EntryLink[]): Promise { if (!links.length) return {linked: 0}; - const res = await Promise.all(links.map(l => writeEntryLink(cat, opts, l))); + const res = await mapConcurrent( + links, WRITE_CONCURRENCY, l => writeEntryLink(cat, opts, l)); const firstErr = res.find(r => r.error); if (firstErr) return {linked: 0, error: firstErr.error}; return {linked: links.length}; @@ -686,11 +693,6 @@ function planSummary( } -// The id segment of a full entry/entryType resource name (after the last '/'). -function idOf(name: string): string { - return name.split('/').pop() ?? name; -} - function isOk(res: {status: number}): boolean { return res.status === 200; } @@ -715,3 +717,184 @@ function isPropagating(res: {message?: string}): boolean { function errText(res: {status: number; message?: string}): string { return res.message?.trim() || `HTTP ${res.status}`; } + + +// --------------------------------------------------------------------------- +// Pull: Knowledge Catalog -> Semantic Model IR. +// +// The read counterpart of deployKnowledgeCatalog and the inverse of push. +// Unlike a write, a pull needs no server-side type provisioning -- only that +// the `semantic-*` entries exist. It enumerates the entry group, keeps the +// semantic entries, hydrates each one's aspect data (a BASIC list omits aspect +// data, so each entry is re-fetched with its aspect types -- an entity needs +// BOTH its `semantic-entity` aspect and the built-in `schema` aspect), and +// hands the hydrated entries to the pure reader (modelsFromCatalogResources). +// --------------------------------------------------------------------------- + +export interface KcPullOptions { + project: string; + location: string; + entryGroup: string; + model?: string; // limit to a single model by name (default: all) +} + +export interface KcPullResult { + models: SemanticModel[]; + warnings: string[]; +} + +// Upper bound on in-flight aspect-hydration fetches during a pull. +const HYDRATE_CONCURRENCY = 8; + +// Reads the semantic models back from a Knowledge Catalog entry group. Emits no +// console output; warnings (skipped entries, no match for --model, reader +// warnings) are returned for the caller to print. +export async function pullKnowledgeCatalog( + cat: CatalogClient, opts: KcPullOptions): Promise { + const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const warnings: string[] = []; + + // Enumerate the group (paging is inherently sequential) and pick the semantic + // entries, then hydrate their aspects concurrently: a BASIC list omits aspect + // data, so each entry needs its own lookupEntry, and those fetches are + // independent. The pool preserves input order so warnings stay deterministic. + const targets: {entry: Entry; aspectTypes: string[]}[] = []; + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const aspectTypes = semanticAspectTypes(entry.entryType); + if (aspectTypes) targets.push({entry, aspectTypes}); + // else: not part of a semantic model; ignore it. + } + + // When scoped to one model, hydrate only that model's entries -- its anchor + // (matched by name) plus the children pointing at it. A list already carries + // entrySource + parentEntry, so this avoids fetching every other model's + // aspects. No match short-circuits with just the not-found warning. + let scoped = targets; + if (opts.model) { + scoped = scopeToModel(targets, opts.model); + if (!scoped.length) { + return { + models: [], + warnings: [ + `no semantic model named '${opts.model}' found in ${destination}` + ], + }; + } + } + + const fetched = await mapConcurrent( + scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry '${entry.name}' (status ${ + res.status}); skipped` + }; + } + return {entry: res.result}; + }); + + const hydrated: Entry[] = []; + for (const r of fetched) { + if (r.entry) + hydrated.push(r.entry); + else if (r.warning) + warnings.push(r.warning); + } + + const read = modelsFromCatalogResources(hydrated); + warnings.push(...read.warnings); + + // Defense in depth: keep only the requested model even if the reader surfaced + // another anchor (e.g. a child whose parentEntry pointed outside the scope). + let models = read.models; + if (opts.model) { + models = models.filter(m => m.name === opts.model); + if (!models.length) { + warnings.push( + `no semantic model named '${opts.model}' found in ${destination}`); + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// The aspect type resource names to hydrate for a semantic entry, derived from +// its entryType (the aspect types are the parallel resources in the same +// project/location). An entity carries two aspects: its `semantic-entity` +// aspect and the built-in `schema` aspect that holds its fields. Returns +// undefined for entries that are not part of a semantic model. +function semanticAspectTypes(entryType: string): string[]|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + const typeBase = entryType.slice(0, idx); + const t = entryType.slice(idx + marker.length); + const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + switch (t) { + case 'semantic-model': + return [aspectType('semantic-model')]; + case 'semantic-entity': + return [aspectType('semantic-entity'), aspectType('schema')]; + case 'semantic-metric': + return [aspectType('semantic-metric')]; + default: + return undefined; + } +} + + +// Restricts hydration targets to a single model: the semantic-model anchor +// whose name (entrySource.displayName, else the entry id) matches `model`, plus +// every child entry whose parentEntry is that anchor. Uses only list-level +// fields (no aspect data), so it runs before hydration and avoids fetching +// unrelated models' aspects. Returns [] when no anchor matches. +function scopeToModel( + targets: {entry: Entry; aspectTypes: string[]}[], + model: string): {entry: Entry; aspectTypes: string[]}[] { + const isAnchor = (t: {entry: Entry}) => + !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); + const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = new Set( + targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === + model) + .map(t => t.entry.name)); + if (!matchedAnchorNames.size) return []; + // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds + // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a + // project-id normalization mismatch) still belongs to it. Without this a + // scoped pull would drop children a full pull keeps. + const soleAnchor = + allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; + const soleMatched = + soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); + return targets.filter( + t => matchedAnchorNames.has(t.entry.name) || + matchedAnchorNames.has(t.entry.parentEntry ?? '') || + (soleMatched && !isAnchor(t) && + !allAnchorNames.has(t.entry.parentEntry ?? ''))); +} + + +// Maps `items` through `fn` with at most `limit` calls in flight, returning +// results in input order (so downstream ordering stays deterministic). +async function mapConcurrent( + items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker(): Promise { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + } + const workers = + Array.from({length: Math.min(limit, items.length)}, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index a3051b22..5e5860c9 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -45,7 +45,8 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {bigQueryGraphTargets} from './deploy_bigquery'; -import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; +import {DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; +import {referencedEntityNames} from './sql_expr_utils'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -539,3 +540,263 @@ function linkSlug(s: string): string { function unquote(part: string): string { return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); } + + +// --------------------------------------------------------------------------- +// Reader: Knowledge Catalog entries -> Semantic Model IR. +// +// The inverse of generateCatalogResources: it reconstructs the IR from the +// entries a pull hydrated (see deploy_knowledge_catalog.pullKnowledgeCatalog). +// `semantic-entity` / `semantic-metric` entries are grouped under their +// `semantic-model` anchor via `parentEntry`; entries of other types are +// ignored. Resources are matched by type-name SUFFIX, so a reader need not know +// which system-type project/location the emitter used. +// +// Fidelity is bounded by what the emitter persisted, so this read is the +// inverse of the WRITE, not of the authored document. It recovers names, +// descriptions, data sources, field datatypes (via the schema aspect) and +// DIMENSION roles, field/metric expressions, and each +// metric's attach entity (re-derived from its expression, as the loader does). +// It cannot recover what the emitter does not write: entity keys/unique keys, +// `ai_context`, field labels, `importedDialect`, `custom_extensions`, and +// relationships (the graph edges live in the BigQuery property graph, not the +// catalog). +// --------------------------------------------------------------------------- + +export interface ReadResult { + models: SemanticModel[]; + warnings: string[]; +} + +/** + * Reconstructs the Semantic Model IR from Knowledge Catalog entries. + * + * Returns one model per `semantic-model` anchor plus any warnings (no anchor, + * an orphaned child, an entry missing its aspect data). Entries must already be + * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a + * BASIC list omits aspect data, so the puller re-fetches each entry first. + */ +export function modelsFromCatalogResources(entries: Entry[]): ReadResult { + const warnings: string[] = []; + + const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); + const entityEntries = + entries.filter(e => semanticType(e) === 'semantic-entity'); + const metricEntries = + entries.filter(e => semanticType(e) === 'semantic-metric'); + + if (!anchors.length) { + warnings.push('no semantic-model entry found; nothing to reconstruct'); + return {models: [], warnings: [...new Set(warnings)]}; + } + + // A child belongs to its anchor by parentEntry. When there is exactly one + // anchor, children whose parentEntry does not resolve (e.g. a project-id + // normalization mismatch) are still attached to it rather than dropped. + const anchorNames = new Set(anchors.map(a => a.name)); + const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; + const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( + e => e.parentEntry === anchorName || + (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + + const models = anchors.map(anchor => { + const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + + const entities = childrenOf(anchor.name, entityEntries) + .map(e => readEntity(e, warnings)); + const entityNames = entities.map(e => e.name); + const metrics = childrenOf(anchor.name, metricEntries) + .map(e => readMetric(e, entityNames, warnings)); + + // Relationships are not published to the catalog (see the file header), so + // a reconstructed model always has an empty edge set. + const model: SemanticModel = {name, entities, relationships: [], metrics}; + const description = anchor.entrySource?.description; + if (description !== undefined) model.description = description; + return model; + }); + + // Flag children that resolved to no anchor at all (only possible with + // multiple anchors, where the sole-anchor fallback does not apply). + if (!soleAnchor) { + for (const child of [...entityEntries, ...metricEntries]) { + if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { + warnings.push(`entry '${ + child.name}' has no resolvable parent semantic-model; omitted`); + } + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// Reconstructs an entity from its `semantic-entity` aspect (the backing source) +// and the built-in `schema` aspect (its fields). Keys are not persisted by the +// emitter and so come back empty. +function readEntity(entry: Entry, warnings: string[]): Entity { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const semantic = aspectData(entry, 'semantic-entity'); + const schema = aspectData(entry, 'schema'); + if (!Object.keys(semantic).length) { + warnings.push(`entity '${ + name}': no semantic-entity aspect data (fetch with the aspect type)`); + } + + const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); + if (!dataSource) { + warnings.push( + `entity '${name}': no backing data source in the semantic-entity ` + + `aspect; 'source' will be empty and the entity may not load`); + } + const entity: Entity = { + name, + dataSource, + keys: [], // not persisted by the emitter; unrecoverable on read + fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + }; + const description = entry.entrySource?.description; + if (description !== undefined) entity.description = description; + return entity; +} + + +// Reconstructs a field from one `schema` aspect field record, inverting +// schemaAspectData: the datatype from dataType/metadataType, expressions from +// the nested `semantics` block, and the DIMENSION role back to a dimension +// marker. +function readField(fd: any, entityName: string, warnings: string[]): Field { + const name = fd?.name; + if (name === undefined || name === '') { + warnings.push( + `entity '${entityName}': a schema field is missing its name; the ` + + `field may not load`); + } + const field: Field = {name}; + const sem = fd?.semantics ?? {}; + if (sem.expression !== undefined) field.expression = sem.expression; + const type = irDataType(fd?.dataType, fd?.metadataType); + if (type !== undefined) field.type = type; + if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.description !== undefined) field.description = fd.description; + return field; +} + + +// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` +// is re-derived from the expression (as the loader does) rather than read from +// the aspect, so it stays consistent with the reconstructed entity set. +function readMetric( + entry: Entry, entityNames: string[], warnings: string[]): Metric { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + if (data.expression === undefined) { + warnings.push(`metric '${name}': no expression in semantic-metric aspect`); + } + + const metric: Metric = {name}; + if (data.expression !== undefined) metric.expression = data.expression; + const exprForRefs = data.expression ?? ''; + const referenced = referencedEntityNames(exprForRefs, entityNames); + if (referenced.length === 1) { + metric.entity = referenced[0]; + } else if (exprForRefs && !referenced.length) { + // Parity with the loader's convertMetric: an expression that qualifies no + // known entity is flagged as potentially unplaceable downstream. + warnings.push( + `metric '${name}': expression references no known entity; it may not ` + + `be placeable downstream`); + } + // The emitter writes a required dataType, defaulting a typeless metric to + // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric + // authored without a datatype round-trips as an explicit Decimal rather than + // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) + const type = irDataType(data.dataType, undefined); + if (type !== undefined) metric.type = type; + const description = entry.entrySource?.description; + if (description !== undefined) metric.description = description; + return metric; +} + + +// The inverse of columnDataType/columnMetadataType: maps the schema aspect's +// dataType (disambiguated by metadataType only for the STRING family) back to +// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read +// as un-typed (undefined) -- the loader's default -- since the emitter cannot +// distinguish an authored `String` from an un-typed field (both emit STRING). +function irDataType(dataType: string|undefined, metadataType: string|undefined): + DataType|undefined { + switch (dataType) { + case 'INT64': + return 'Integer'; + case 'NUMERIC': + return 'Decimal'; + case 'FLOAT64': + return 'Float'; + case 'BOOL': + return 'Boolean'; + case 'DATE': + return 'Date'; + case 'TIME': + return 'Time'; + case 'DATETIME': + return 'DateTime'; + case 'TIMESTAMP': + return 'DateTimeTz'; + case 'STRING': + return metadataType === 'OTHER' ? 'Opaque' : undefined; + default: + return undefined; + } +} + + +// The inverse of resourcePath: a BigQuery linked-resource URI becomes the +// canonical `project.dataset.table` string; anything else (a verbatim query or +// a passthrough reference) is returned unchanged. +function dataSourceFromResource(resource: string|undefined): string { + const value = (resource ?? '').trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + + +// The bare `semantic-*` type of an entry, matched by entryType suffix so the +// system-type project/location need not be known. Returns undefined for entries +// that are not part of a semantic model. +function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| + 'semantic-metric'|undefined { + for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as + const) { + if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; + } + return undefined; +} + + +// The `data` payload of an entry's aspect of the given bare type, matched by +// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` +// suffix (robust to whichever system-type project/location the emitter used). +// Returns an empty object when the aspect is absent. +function aspectData(entry: Entry, type: string): Record { + const aspects = entry.aspects ?? {}; + for (const [key, aspect] of Object.entries(aspects)) { + if (key.endsWith(`.${type}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + + +// The id segment of a full entry resource name (after the last '/'). +export function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} diff --git a/toolbox/mdcode/src/libts/semantic/serialize.ts b/toolbox/mdcode/src/libts/semantic/serialize.ts new file mode 100644 index 00000000..dd215100 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/serialize.ts @@ -0,0 +1,256 @@ +// Serializes the Semantic Model IR (./ir) back to the open AI-first semantics +// format (YAML). This is the inverse of `loader.ts`: `loader` reads authored +// YAML into the IR; this module writes the IR out as a YAML document the loader +// can read back. It is the local-workspace sink for `pull` (Knowledge Catalog +// -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR -> a +// destination. +// +// Fidelity is at the IR level, not byte-for-byte with a hand-authored file. The +// loader normalizes several authoring conveniences into the IR at load time, so +// they are already gone before serialization and cannot be reproduced here: +// * per-dialect `expression.dialects[]` variants collapse to at most two +// forms +// (a target/canonical `expression` + an `importedExpression`); only those +// are re-emitted, each under a single dialect label. +// * comments and key ordering are not preserved. +// What the loader DOES keep on the IR round-trips here: names, descriptions, +// `ai_context` (instructions / synonyms / examples), `custom_extensions` +// (verbatim), keys and unique keys, data sources, field datatypes / labels / +// dimension flags, expressions, and relationship join columns. See +// serialize.test.ts. +// +// An `association` (junction-table) relationship has no open-format syntax (the +// loader cannot produce one), so only its direct foreign-key view (from/to + +// columns) is serialized; the junction detail is dropped with a note. + +import * as yaml from 'yaml'; + +import {AiContext, CustomExtension, Entity, Field, Metric, Relationship, SemanticModel,} from './ir'; + +// The schema version the loader was written against; re-emitted verbatim so a +// serialized document loads without a version-mismatch warning. Mirrors +// loader.SUPPORTED_VERSION. +const SERIALIZED_VERSION = '0.2.0.dev0'; + +// The dialect label for the IR's target/canonical `expression`. The IR does not +// record which authored dialect that string came from (the loader picked it +// from the target dialect or the ANSI_SQL fallback and discarded the label), +// and its contract is "GoogleSQL-valid". BIGQUERY is the loader's default +// target dialect, so labeling the canonical form BIGQUERY makes the loader +// re-pick it exactly on reload -- a clean round trip with no dialect-fallback +// note. +const CANONICAL_DIALECT = 'BIGQUERY'; + +// The dialect label used for an `importedExpression` whose `importedDialect` +// was lost (e.g. read back from a Knowledge Catalog aspect that does not +// persist the dialect). A non-target, non-canonical label so the loader +// re-reads it as the imported vendor form rather than the canonical expression. +const UNKNOWN_IMPORTED_DIALECT = 'IMPORTED'; + +export interface SerializeResult { + yaml: string; + warnings: string[]; +} + +/** + * Serializes a single semantic model to a YAML document string in the open + * AI-first semantics format. The document contains exactly this one model; + * `pull` writes one file per model + * (catalog/EntryGroups//.yaml). + * + * Warnings flag IR content that has no loadable representation (an association + * relationship's junction detail), so the caller can surface the lossy edge. + */ +export function serializeModel(model: SemanticModel): SerializeResult { + const warnings: string[] = []; + const text = yaml.stringify(modelDocument(model, warnings)); + return {yaml: text, warnings: [...new Set(warnings)]}; +} + +// Builds the plain document object (version + one model) that yaml.stringify +// renders. Kept separate so tests can assert the structure without parsing +// YAML. +export function modelDocument( + model: SemanticModel, warnings: string[] = []): Record { + return { + version: SERIALIZED_VERSION, + semantic_model: [modelDoc(model, warnings)], + }; +} + +function modelDoc( + model: SemanticModel, warnings: string[]): Record { + // `datasets` is required (min 1) by the loader. A reconstructed model with no + // entities (e.g. every entity fetch failed during a pull) would serialize to + // a document the loader rejects; emit the (empty) array but flag it so the + // lossy edge is visible rather than surfacing later as an opaque load error. + const datasets = (model.entities ?? []).map(e => datasetDoc(e, warnings)); + if (!datasets.length) { + warnings.push( + `model '${model.name}': no datasets (entities); the document requires ` + + `at least one and will not load until an entity is present.`); + } + return compact({ + name: model.name, + description: model.description, + ai_context: aiContextDoc(model.aiContext), + custom_extensions: customExtensionsDoc(model.customExtensions), + datasets, + relationships: nonEmpty( + (model.relationships ?? []).map(r => relationshipDoc(r, warnings))), + metrics: nonEmpty((model.metrics ?? []).map(m => metricDoc(m, warnings))), + }); +} + +function datasetDoc( + entity: Entity, warnings: string[]): Record { + return compact({ + name: entity.name, + source: entity.dataSource, + primary_key: nonEmpty(entity.keys), + unique_keys: nonEmpty(entity.uniqueKeys), + description: entity.description, + ai_context: aiContextDoc(entity.aiContext), + fields: nonEmpty((entity.fields ?? []).map(f => fieldDoc(f, warnings))), + custom_extensions: customExtensionsDoc(entity.customExtensions), + }); +} + +function fieldDoc(field: Field, warnings: string[]): Record { + const expression = expressionDoc( + field.expression, field.importedExpression, field.importedDialect); + if (!expression) { + warnings.push( + `field '${field.name}': no expression; the loader requires one per ` + + `field and the document will not load until it is set.`); + } + return compact({ + name: field.name, + expression, + datatype: field.type, + label: field.label, + dimension: dimensionDoc(field), + description: field.description, + ai_context: aiContextDoc(field.aiContext), + custom_extensions: customExtensionsDoc(field.customExtensions), + }); +} + +function metricDoc(metric: Metric, warnings: string[]): Record { + // `entity` is derived by the loader from the expression's entity qualifiers, + // so it is intentionally not emitted: the loader recomputes it on reload. + const expression = expressionDoc( + metric.expression, metric.importedExpression, metric.importedDialect); + if (!expression) { + warnings.push( + `metric '${metric.name}': no expression; the loader requires one per ` + + `metric and the document will not load until it is set.`); + } + return compact({ + name: metric.name, + expression, + datatype: metric.type, + description: metric.description, + ai_context: aiContextDoc(metric.aiContext), + custom_extensions: customExtensionsDoc(metric.customExtensions), + }); +} + +// Inverts loader.convertRelationship: `from`/`to` are the endpoint entities and +// `from_columns`/`to_columns` are their positional join columns. An association +// (junction-table) edge has no open-format syntax, so only this direct-FK view +// is emitted and the junction detail is flagged. +function relationshipDoc( + rel: Relationship, warnings: string[]): Record { + if (rel.association) { + warnings.push( + `relationship '${ + rel.name}': association (junction-table) detail has no ` + + `open-format representation and is not serialized; only its foreign-key ` + + `endpoints are written.`); + } + return compact({ + name: rel.name, + from: rel.source.entity, + to: rel.destination.entity, + from_columns: nonEmpty(rel.source.columns), + to_columns: nonEmpty(rel.destination.columns), + description: rel.description, + ai_context: aiContextDoc(rel.aiContext), + custom_extensions: customExtensionsDoc(rel.customExtensions), + }); +} + +// Renders the `expression` object matching the loader's schema (a `dialects` +// array of {dialect, expression}). Emits the target/canonical form under +// CANONICAL_DIALECT and the imported vendor form under its own dialect, so the +// loader re-picks each into the same IR field. Returns undefined when neither +// form is present (the loader requires an expression, but a pathological field +// with none is dropped rather than fabricated). +function expressionDoc( + expression: string|undefined, importedExpression: string|undefined, + importedDialect: string|undefined): Record|undefined { + const dialects: {dialect: string; expression: string}[] = []; + if (expression !== undefined) { + dialects.push({dialect: CANONICAL_DIALECT, expression}); + } + if (importedExpression !== undefined) { + let label = importedDialect ?? UNKNOWN_IMPORTED_DIALECT; + // Never emit two dialect entries under the same label: the loader would + // pick between them non-deterministically. If the imported form's dialect + // collides with the canonical label already pushed, fall back to the + // imported placeholder. + if (expression !== undefined && label === CANONICAL_DIALECT) { + label = UNKNOWN_IMPORTED_DIALECT; + } + dialects.push({dialect: label, expression: importedExpression}); + } + return dialects.length ? {dialects} : undefined; +} + +// Emits the field's `dimension` block when present, inverting +// loader.convertField (which sets `dimension = {}` for a bare marker and copies +// `is_time`). The key is emitted whenever the IR carries dimension metadata, +// even for an empty marker, so a dimension field reloads as a dimension. +function dimensionDoc(field: Field): Record|undefined { + if (!field.dimension) return undefined; + return compact({is_time: field.dimension.isTime}); +} + +// Emits the structured `ai_context` (the only authoring path the loader reads +// synonyms/instructions/examples from), inverting loader.normalizeAiContext. +// Returns undefined when the context carries nothing, so `compact` drops the +// key. +function aiContextDoc(ai: AiContext|undefined): Record|undefined { + if (!ai) return undefined; + const doc = compact({ + instructions: ai.instructions, + synonyms: nonEmpty(ai.synonyms), + examples: nonEmpty(ai.examples), + }); + return Object.keys(doc).length ? doc : undefined; +} + +// Emits vendor `custom_extensions` verbatim (`vendorName` -> `vendor_name`), +// inverting loader.toCustomExtensions. Returns undefined when there are none. +function customExtensionsDoc(exts: CustomExtension[]|undefined): + Record[]|undefined { + if (!exts || !exts.length) return undefined; + return exts.map(e => ({vendor_name: e.vendorName, data: e.data})); +} + +// Drops undefined-valued keys so the emitted YAML only shows fields the model +// actually set (matching the emitters' `compact` convention). +function compact>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +// Returns the array, or undefined when empty/absent, so an empty list is +// omitted rather than rendered as `[]`. +function nonEmpty(items: T[]|undefined): T[]|undefined { + return items && items.length ? items : undefined; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index bd8cf696..59ce84e1 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -14,6 +14,7 @@ import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; import {BigQueryClient} from '../libts/gcp/bigquery'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; +import { serializeModel } from '../libts/semantic/serialize'; import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; @@ -158,15 +159,20 @@ export async function init(options: InitOptions): Promise { } -export async function pull(): Promise { +export interface PullOptions { + // Reconstruct + report only; never writes a file. Mirrors push --validate-only. + dryRun?: boolean; + // Limit the pull to a single model by name (default: all in the entry group). + model?: string; +} + + +export async function pull(options: PullOptions = {}): Promise { const ctx = context.ApiContext.default(); const snapshot = await kcmd.CatalogSnapshot.fromPath('.', ctx); if (snapshot.manifest.source.type === Sources.SEMANTIC_MODEL) { - console.log( - 'Semantic-model scope: nothing to pull. Knowledge Catalog resource ' + - 'pull for the semantic model is not yet implemented.'); - return 0; + return await pullSemanticModel(ctx, snapshot, options); } const catalog = new dataplex.CatalogClient(ctx); @@ -360,3 +366,77 @@ async function pushKnowledgeCatalog( unlinked}.`); return 0; } + + +// Pulls the semantic model's Knowledge Catalog entries back into local model +// documents (catalog/EntryGroups//.yaml) and prints the +// result. The destination coordinates come from the scope +// (project.location.entryGroup). Overwrite policy matches the core pull: +// last-write-wins, local-only documents are left untouched (never deleted). +// Returns a process exit code (0 on success). +async function pullSemanticModel( + ctx: context.ApiContext, snapshot: kcmd.CatalogSnapshot, + options: PullOptions): Promise { + // The semantic-model source always resolves to the SemanticModel layout + // (see createLayout), so these casts are safe. + const layout = snapshot.layout as SemanticModelLayout; + const source = snapshot.manifest.source as SemanticModelSource; + + console.log(options.dryRun + ? 'Reconstructing semantic model from Knowledge Catalog (dry run)...' + : 'Pulling semantic model from Knowledge Catalog...'); + + const catalog = new dataplex.CatalogClient(ctx); + const result = await kc.pullKnowledgeCatalog(catalog, { + project: source.project, + location: source.location, + entryGroup: source.entryGroup, + model: options.model, + }); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + + if (!result.models.length) { + console.log('No semantic models found; nothing to pull.'); + return 0; + } + + let created = 0; + let updated = 0; + // Guard against two reconstructed models whose names map to the same file + // (path-separator sanitizing, or two anchors sharing a display name): the + // later write would silently clobber the earlier. Track written paths so the + // collision is reported and the dry-run/real counts agree on the repeat. + const writtenBy = new Map(); + for (const model of result.models) { + const serialized = serializeModel(model); + for (const w of serialized.warnings) { + console.warn(`Warning: [${model.name}] ${w}`); + } + const target = layout.modelPath(model.name); + const prior = writtenBy.get(target); + if (prior !== undefined && prior !== model.name) { + console.warn( + `Warning: models '${prior}' and '${model.name}' both map to ` + + `${target}; the later overwrites the earlier -- rename one model.`); + } + const existed = writtenBy.has(target) || layout.hasModel(model.name); + writtenBy.set(target, model.name); + if (options.dryRun) { + console.log(` would ${existed ? 'update' : 'create'} ${target}`); + } + else { + layout.writeModelDocument(model.name, serialized.yaml); + console.log(` ${existed ? 'updated' : 'created'} ${target}`); + } + if (existed) updated++; + else created++; + } + + console.log(options.dryRun + ? `Dry run: would write ${created} new and ${updated} updated model document(s).` + : `Wrote ${created} new and ${updated} updated model document(s).`); + return 0; +} diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index cbc67e95..84cceff9 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -25,10 +25,12 @@ cli.command('init', 'Initialize a new catalog snapshot') cli.command('pull', 'Pull catalog entries') - .action(async () => { + .option('--dry-run', 'Reconstruct and report only; do not write files (semantic-model scope)') + .option('--model ', 'Limit the pull to a single model by name (semantic-model scope)') + .action(async (options) => { let exitCode = 1; try { - exitCode = await commands.pull(); + exitCode = await commands.pull(options); } catch (err: any) { console.error('Error:', err.message || err); diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts new file mode 100644 index 00000000..e31a2f94 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts @@ -0,0 +1,244 @@ +// Tests for the semantic-model Knowledge Catalog pull leg +// (pullKnowledgeCatalog in src/libts/semantic/deploy_knowledge_catalog.ts). +// +// pullKnowledgeCatalog is the orchestration around the pure reader: enumerate +// the entry group, hydrate each semantic entry's aspect data, and reconstruct +// the IR. The catalog client is stubbed so no network call is made. The entries +// the fake serves are produced by the real emitter, so this exercises the true +// list -> hydrate -> read path end to end (an entity is re-fetched with BOTH +// its semantic-entity and schema aspects). The reader's own mapping is covered +// in knowledge_catalog.read.test.ts; the focus here is the fetch SEQUENCE: +// aspect hydration, the --model filter, skipped entries, and ignoring foreign +// entries. + +import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; + +import {ApiResult} from '../../../src/libts/gcp/api'; +import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +const SALES: SemanticModel = { + name: 'sales', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [ + {name: 'o_totalprice', expression: 'orders.o_totalprice', type: 'Decimal'} + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + // Metrics carry a datatype through KC (semantic-metric.dataType is + // required); Decimal -> NUMERIC on emit, NUMERIC -> Decimal on read. + type: 'Decimal' + }], +}; + +// The entries the emitter would have written for a model. +function entriesFor(model: SemanticModel): Entry[] { + return generateCatalogResources(model, OPTS).entries; +} + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +// Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, +// or 404s an unknown name). `lookupFail` forces a failure for one entry name. +function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): + {list: any; lookup: any} { + const byName = new Map(served.map(e => [e.name, e])); + const list = spyOn(CatalogClient.prototype, 'listEntries') + .mockImplementation(async function*() { + for (const e of listed) yield e; + } as any); + const lookup = spyOn(CatalogClient.prototype, 'lookupEntry') + .mockImplementation(async (_p, _l, name) => { + if (name === lookupFail) return err(500, 'boom'); + const e = byName.get(name); + return e ? ok(e) : err(404, 'not found'); + }); + return {list, lookup}; +} + +afterEach(() => { + mock.restore(); +}); + + +describe('pullKnowledgeCatalog: happy path', () => { + test('reconstructs the model from listed + hydrated entries', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models).toHaveLength(1); + expect(models[0]).toEqual(SALES); + expect(warnings).toHaveLength(0); + // Every listed semantic entry (model + entity + metric) was hydrated. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + 'an entity is hydrated with BOTH its semantic-entity and schema aspects', + async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + await pullKnowledgeCatalog(cat, OPTS); + + // Find the lookup call for the entity entry and inspect its requested + // aspect types (the 4th arg). + const entityEntry = + entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + const call = + lookup.mock.calls.find((c: any[]) => c[2] === entityEntry.name)!; + const aspectTypes: string[] = call[3]; + expect( + aspectTypes.some(t => t.endsWith('/aspectTypes/semantic-entity'))) + .toBe(true); + expect(aspectTypes.some(t => t.endsWith('/aspectTypes/schema'))) + .toBe(true); + }); +}); + + +describe('pullKnowledgeCatalog: filtering and robustness', () => { + test('--model keeps only the named model', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + }); + + test('--model with no match returns nothing and warns', async () => { + const entries = entriesFor(SALES); + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); + + test( + 'a non-semantic entry in the group is ignored, not fetched', async () => { + const entries = entriesFor(SALES); + const foreign: Entry = { + name: `projects/dest/locations/us/entryGroups/eg/entries/foreign`, + entryType: 'projects/x/locations/us/entryTypes/some-other-type', + }; + const {lookup} = stubClient([...entries, foreign], entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + expect(models).toHaveLength(1); + // The foreign entry is never hydrated (only the 3 semantic entries + // are). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('a failed hydration is skipped with a warning', async () => { + const entries = entriesFor(SALES); + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + stubClient(entries, entries, metricEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + // The model + entity still reconstruct; only the metric is dropped. + expect(models).toHaveLength(1); + expect(models[0].metrics).toHaveLength(0); + expect(warnings.some( + w => /failed to fetch/i.test(w) && w.includes(metricEntry.name))) + .toBe(true); + }); + + test('--model hydrates only the target model\'s entries', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + // SALES has 3 entries (model + entity + metric); inventory's are never + // fetched -- the flag scopes hydration, not just the final result. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + '--model keeps a child whose parentEntry does not resolve (sole-anchor ' + + 'fallback)', + async () => { + const entries = entriesFor(SALES); + // Simulate a project-id normalization mismatch: the metric points at an + // anchor name that differs from the emitted one. With a single model in + // the group a full pull still attaches it via the reader's sole-anchor + // fallback, so a scoped pull must keep it too (else --model silently + // drops a child a full pull returns). + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + metricEntry.parentEntry = + metricEntry.parentEntry!.replace('projects/dest/', 'projects/12345/'); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + + expect(models).toHaveLength(1); + expect(models[0].metrics.map(m => m.name)).toEqual(['total_revenue']); + // The metric was hydrated despite the parent mismatch (3 entries). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('--model with no match fetches nothing and warns', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(lookup).not.toHaveBeenCalled(); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts new file mode 100644 index 00000000..a8d5e447 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts @@ -0,0 +1,258 @@ +// Behavior specification for the Knowledge Catalog reader +// (modelsFromCatalogResources in src/libts/semantic/knowledge_catalog.ts). +// +// The reader is the inverse of the emitter (generateCatalogResources). The +// central guarantee is an emitter -> reader round trip: emit a model's entries, +// read them back, and get an IR equal to the source WHERE the emitter is +// lossless. The write drops content by design (entity keys, ai_context, field +// labels, importedDialect, relationships -- see the emitter header), so the +// expected read-back is the source model with exactly those fields cleared. +// Targeted tests pin the mapping details a round trip cannot isolate (the +// dataType inverse, the DIMENSION role, resource-URI parsing, metric attach +// re-derivation, and parent/anchor grouping). + +import {describe, expect, test} from 'bun:test'; + +import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources, modelsFromCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +// Emits a model to entries and reads it straight back. +function roundTrip(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources(entries); +} + + +describe('emitter -> reader round trip (lossless slice)', () => { + // A model using only round-trippable content: no keys/ai_context/labels/ + // relationships (all dropped by the write), and datatypes that invert + // cleanly. + const source: SemanticModel = { + name: 'sales', + description: 'the sales model', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], // keys are not persisted; keep empty so the round trip matches + fields: [ + {name: 'o_orderkey', expression: 'orders.o_orderkey', type: 'Integer'}, + { + name: 'o_orderdate', + expression: 'orders.o_orderdate', + type: 'Date', + dimension: {}, + description: 'order date', + }, + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + type: 'Decimal', + }], + }; + + test('reconstructs an IR equal to the source', () => { + const {models} = roundTrip(source); + expect(models).toHaveLength(1); + expect(models[0]).toEqual(source); + }); +}); + + +describe('dataType inverse (schema aspect -> IR type)', () => { + // Emit a one-field model of each IR type, read it back, and check the field's + // reconstructed type. String and Opaque both emit dataType STRING; String + // (indistinguishable from an un-typed field) reads back as undefined, while + // Opaque is disambiguated by metadataType OTHER. + const cases: [Metric['type']|undefined, Metric['type']|undefined][] = [ + ['Integer', 'Integer'], + ['Decimal', 'Decimal'], + ['Float', 'Float'], + ['Boolean', 'Boolean'], + ['Date', 'Date'], + ['Time', 'Time'], + ['DateTime', 'DateTime'], + ['DateTimeTz', 'DateTimeTz'], + ['Opaque', 'Opaque'], + ['String', undefined], // collapses to un-typed + [undefined, undefined], // un-typed stays un-typed + ]; + + for (const [type, expected] of cases) { + test(`${type ?? 'un-typed'} -> ${expected ?? 'un-typed'}`, () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f', ...(type ? {type} : {})}], + }], + relationships: [], + metrics: [], + }; + const back = roundTrip(model).models[0].entities[0].fields[0]; + expect(back.type).toBe(expected as any); + }); + } +}); + + +describe('field mapping details', () => { + function readField(field: Entity['fields'][number]) { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: [field]}], + relationships: [], + metrics: [], + }; + return roundTrip(model).models[0].entities[0].fields[0]; + } + + test('a DIMENSION role reads back as a dimension marker', () => { + const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toEqual({}); + }); + + test('a non-dimension field has no dimension marker', () => { + const back = readField({name: 'f', expression: 'e.f'}); + expect(back.dimension).toBeUndefined(); + }); + + test( + 'an imported expression is not persisted to or recovered from the catalog', + () => { + const back = readField({ + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }); + expect(back.expression).toBe('e.amt'); + // The emitter no longer writes importedExpression/importedDialect, so + // neither survives the round trip. + expect(back.importedExpression).toBeUndefined(); + expect(back.importedDialect).toBeUndefined(); + }); +}); + + +describe('data source resource-path parsing', () => { + function readDataSource(dataSource: string): string { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource, keys: [], fields: []}], + relationships: [], + metrics: [], + }; + return roundTrip(model).models[0].entities[0].dataSource; + } + + test( + 'a three-part BigQuery reference round-trips through the resource URI', + () => { + expect(readDataSource('proj.ds.tbl')).toBe('proj.ds.tbl'); + }); + + test('a verbatim query source is preserved unchanged', () => { + const query = 'SELECT * FROM t'; + expect(readDataSource(query)).toBe(query); + }); +}); + + +describe('metric attach entity is re-derived from the expression', () => { + test( + 'a single-entity metric attaches; a cross-entity metric does not', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + metrics: [ + {name: 'revenue', expression: 'SUM(orders.amt)', entity: 'orders'}, + { + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)' + }, + ], + }; + const {models} = roundTrip(model); + const byName = new Map(models[0].metrics.map(m => [m.name, m])); + expect(byName.get('revenue')!.entity).toBe('orders'); + expect(byName.get('mix')!.entity).toBeUndefined(); + }); +}); + + +describe('anchor / parent grouping', () => { + test('no semantic-model entry yields no models and a warning', () => { + const {models, warnings} = modelsFromCatalogResources([]); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic-model entry/i.test(w))).toBe(true); + }); + + test('two models keep their own children by parentEntry', () => { + const a = generateCatalogResources( + { + name: 'a', + entities: [{name: 'ea', dataSource: 'p.d.a', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const b = generateCatalogResources( + { + name: 'b', + entities: [{name: 'eb', dataSource: 'p.d.b', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const {models} = modelsFromCatalogResources([...a.entries, ...b.entries]); + const byName = new Map(models.map(m => [m.name, m])); + expect(byName.get('a')!.entities.map(e => e.name)).toEqual(['ea']); + expect(byName.get('b')!.entities.map(e => e.name)).toEqual(['eb']); + }); +}); + + +describe('metric expression referencing no known entity', () => { + test('warns that the metric may be unplaceable and leaves it unattached', + () => { + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], + relationships: [], + // References `widgets`, which is not an entity of this model. + metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].metrics[0].entity).toBeUndefined(); + expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts new file mode 100644 index 00000000..ac4c7bc0 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts @@ -0,0 +1,75 @@ +// Tests for the SemanticModel layout's pull write-path +// (src/libts/layouts/semantic-model.ts): modelPath / hasModel / +// writeModelDocument. These are the sink `pull` writes reconstructed models to; +// the push-side discovery (modelDocuments) is exercised via the deploy tests. + +import {afterEach, beforeEach, describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import {SemanticModelLayout} from '../../../src/libts/layouts/semantic-model'; + +let root: string; +let catalogPath: string; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'kcmd-layout-')); + catalogPath = path.join(root, 'catalog'); +}); + +afterEach(() => { + fs.rmSync(root, {recursive: true, force: true}); +}); + +async function layout(entryGroup?: string): Promise { + const l = new SemanticModelLayout(catalogPath, entryGroup); + await l.init(); + return l; +} + + +describe('SemanticModelLayout write path', () => { + test('modelPath maps to EntryGroups//.yaml', async () => { + const l = await layout('eg'); + expect(l.modelPath('sales')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'sales.yaml')); + }); + + test('modelPath sanitizes path separators in the model name', async () => { + const l = await layout('eg'); + expect(l.modelPath('a/b')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'a_b.yaml')); + }); + + test('modelPath throws without an entry group', async () => { + const l = await layout(undefined); + expect(() => l.modelPath('sales')).toThrow(/entry group/i); + }); + + test( + 'writeModelDocument creates the file, dirs, and indexes it', async () => { + const l = await layout('eg'); + expect(l.hasModel('sales')).toBe(false); + + l.writeModelDocument('sales', 'version: x\n'); + + const p = l.modelPath('sales'); + expect(fs.existsSync(p)).toBe(true); + expect(fs.readFileSync(p, 'utf8')).toBe('version: x\n'); + expect(l.hasModel('sales')).toBe(true); + // Indexed, so a subsequent read surfaces it as a model document. + expect(l.modelDocuments()).toEqual([ + {name: 'sales', text: 'version: x\n'} + ]); + }); + + test( + 'writeModelDocument overwrites an existing document (last-write-wins)', + async () => { + const l = await layout('eg'); + l.writeModelDocument('sales', 'first\n'); + l.writeModelDocument('sales', 'second\n'); + expect(fs.readFileSync(l.modelPath('sales'), 'utf8')).toBe('second\n'); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts b/toolbox/mdcode/tests/libts/semantic/serialize.test.ts new file mode 100644 index 00000000..81c4454c --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/serialize.test.ts @@ -0,0 +1,290 @@ +// Behavior specification for the semantic-model serializer +// (src/libts/semantic/serialize.ts). +// +// serialize.ts is the inverse of loader.ts: IR -> open-format YAML. The +// strongest guarantee is a round trip through the loader -- load a fixture to +// the IR, serialize it, load the serialized text again, and assert the two IRs +// are identical. That pins IR-level fidelity across every feature the loader +// produces (datasets, fields, datatypes, dimensions, labels, ai_context, +// custom_extensions, relationships, metrics, imported expressions) without +// hard-coding YAML text. Targeted structural tests cover the mapping details a +// round trip cannot isolate. + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as yaml from 'yaml'; + +import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; +import {loadModels} from '../../../src/libts/semantic/loader'; +import {modelDocument, serializeModel} from '../../../src/libts/semantic/serialize'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +function loadFixture(name: string): SemanticModel[] { + const text = fs.readFileSync(path.join(FIXTURES, name), 'utf8'); + return loadModels(text).models; +} + + +describe('loader <-> serialize round trip is IR-stable', () => { + // Each fixture exercises a different slice of the format: relationships + + // ai_context + synonyms + label + dimension; the full tpc-ds corpus with + // custom_extensions + unique_keys; explicit datatypes; and imported + // (vendor-dialect) expressions. + const fixtures = [ + 'star_orders_customer.yaml', + 'tpcds_retail.yaml', + 'sales_google_ext.yaml', + 'vendor_dialects.yaml', + 'lineitem_databricks_ext.yaml', + 'sales_bq_graph_target.yaml', + ]; + + for (const fixture of fixtures) { + test(`${fixture} survives IR -> YAML -> IR unchanged`, () => { + const original = loadFixture(fixture); + expect(original.length).toBeGreaterThan(0); + + for (const model of original) { + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models; + expect(reloaded).toHaveLength(1); + // IR-level equality: every field the loader keeps must match exactly. + expect(reloaded[0]).toEqual(model); + } + }); + } +}); + + +describe('serialized document structure', () => { + const model = loadFixture('star_orders_customer.yaml')[0]; + const doc = modelDocument(model) as any; + const sm = doc.semantic_model[0]; + + test('emits the supported version and a single model', () => { + expect(doc.version).toBe('0.2.0.dev0'); + expect(doc.semantic_model).toHaveLength(1); + expect(sm.name).toBe(model.name); + }); + + test('a dataset source is the opaque dataSource string, verbatim', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.source).toBe(entity.dataSource); + expect(typeof orders.source).toBe('string'); + }); + + test('primary_key mirrors the entity keys', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.primary_key).toEqual(entity.keys); + }); + + test('ai_context is emitted structurally (synonyms round-trip)', () => { + // The fixture annotates a field (o_orderdate) with synonyms; find it and + // assert the structured ai_context is emitted under that field. + const entity = model.entities.find( + e => e.fields.some(f => f.aiContext?.synonyms?.length))!; + const field = entity.fields.find(f => f.aiContext?.synonyms?.length)!; + const dsDoc = sm.datasets.find((d: any) => d.name === entity.name); + const fieldDoc = dsDoc.fields.find((f: any) => f.name === field.name); + expect(fieldDoc.ai_context.synonyms).toEqual(field.aiContext!.synonyms); + }); + + test('a relationship maps to from/to + positional columns', () => { + expect(sm.relationships.length).toBeGreaterThan(0); + const rel = model.relationships[0]; + const relDoc = sm.relationships[0]; + expect(relDoc.from).toBe(rel.source.entity); + expect(relDoc.to).toBe(rel.destination.entity); + expect(relDoc.from_columns).toEqual(rel.source.columns); + expect(relDoc.to_columns).toEqual(rel.destination.columns); + }); +}); + + +describe('expression + datatype + dimension mapping', () => { + test('an explicit datatype round-trips as `datatype`', () => { + const model = loadFixture('sales_google_ext.yaml')[0]; + const typed = model.entities.flatMap(e => e.fields).find(f => f.type); + expect(typed).toBeDefined(); + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models[0]; + const back = reloaded.entities.flatMap(e => e.fields) + .find(f => f.name === typed!.name)!; + expect(back.type).toBe(typed!.type); + }); + + test('a bare dimension marker survives as `dimension: {}`', () => { + const field: + Field = {name: 'ship_date', expression: 'e.ship_date', dimension: {}}; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const fieldDoc = doc.semantic_model[0].datasets[0].fields[0]; + expect(fieldDoc.dimension).toEqual({}); + // And it reloads back to a dimension field. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.entities[0].fields[0].dimension).toEqual({}); + }); + + test('an imported vendor expression is emitted under its own dialect', () => { + const field: Field = { + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const dialects = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects; + const labels = dialects.map((d: any) => d.dialect); + expect(labels).toContain('SNOWFLAKE'); + // The canonical form is labeled BIGQUERY so the loader re-picks it exactly. + expect(labels).toContain('BIGQUERY'); + }); + + test('a metric does not emit its derived attach entity', () => { + const metric: Metric = { + name: 'total', + expression: 'SUM(orders.amt)', + entity: 'orders', + }; + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{name: 'amt', expression: 'orders.amt'}], + }], + relationships: [], + metrics: [metric], + }; + const metricDoc = + (modelDocument(model) as any).semantic_model[0].metrics[0]; + expect(metricDoc).not.toHaveProperty('entity'); + // The loader re-derives it on reload. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.metrics[0].entity).toBe('orders'); + }); +}); + + +describe('lossy edges are flagged', () => { + test( + 'an association relationship warns and drops the junction detail', () => { + const rel: Relationship = { + name: 'enrollment', + source: {entity: 'student', columns: ['id']}, + destination: {entity: 'course', columns: ['id']}, + association: { + dataSource: 'p.d.enrollment', + keys: ['student_id', 'course_id'], + sourceColumns: ['student_id'], + destinationColumns: ['course_id'], + }, + }; + const model: SemanticModel = { + name: 'school', + entities: [ + { + name: 'student', + dataSource: 'p.d.student', + keys: ['id'], + fields: [] + }, + { + name: 'course', + dataSource: 'p.d.course', + keys: ['id'], + fields: [] + }, + ], + relationships: [rel], + metrics: [], + }; + const {yaml: text, warnings} = serializeModel(model); + expect(warnings.some(w => /association/i.test(w))).toBe(true); + // The direct-FK view is still emitted (from/to + columns), so it + // reloads. + const relDoc = yaml.parse(text).semantic_model[0].relationships[0]; + expect(relDoc.from).toBe('student'); + expect(relDoc.to).toBe('course'); + expect(relDoc.from_columns).toEqual(['id']); + }); +}); + + +describe('serialize flags loader-invalid reconstructions', () => { + test('a model with no entities warns that datasets is required', () => { + const model: SemanticModel = { + name: 'empty', + entities: [], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /no datasets/i.test(w))).toBe(true); + }); + + test('a field with no expression warns that one is required', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'orphan'}], + }], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /field 'orphan'.*no expression/i.test(w))) + .toBe(true); + }); + + test( + 'an imported dialect colliding with the canonical label is relabeled', + () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{ + name: 'amt', + expression: 'orders.amt', + importedExpression: 'orders.AMT', + importedDialect: 'BIGQUERY', + }], + }], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const labels: string[] = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects.map( + (d: any) => d.dialect); + // No duplicate dialect label: the imported form is relabeled off + // BIGQUERY so the loader does not pick between two BIGQUERY entries. + expect(new Set(labels).size).toBe(labels.length); + expect(labels).toContain('BIGQUERY'); + }); +}); From 2d7f9bf407609514e4c93d6d308d49f1dba1310e Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:22:44 +0000 Subject: [PATCH 02/12] mdcode: restructure pull into converter-scaffold files; add fixture goldens + docs Addresses PR review feedback on the KC pull leg: - Rename serialize.ts -> osi_converter.ts (the OSI <-> IR converter). The name now says what it converts between; a header banner notes it currently holds only the serialize direction and that the loader migrates in post-#278. - Extract the KC reader into kc_converter.ts and the network pull into pull_kc.ts, so the new capability lives in its own files rather than swelling knowledge_catalog.ts / deploy_knowledge_catalog.ts. Those two files return to their #278 state (emit-only / push-only). This is scaffolding for the eventual two-layer split (pure converters vs push/pull orchestration); the remaining halves move in once #278 merges, with no further file renames. - Reorganize the pull tests around committed golden artifacts: each corpus fixture now has an .osi.golden.yaml (IR -> OSI) and a .pull.golden.yaml (KC entries -> IR -> OSI). A reviewer sees the whole input and output as files and can diff the two to see exactly what a Knowledge Catalog round trip drops. Test files renamed to match their modules (osi_converter/kc_converter/pull_kc). - Document `kcmd pull` in docs/semantic-model.md: the --dry-run/--model flags, multiple models per entry group, last-write-wins overwrite policy, and the catalog-not-a-full-copy round-trip loss. --- toolbox/mdcode/docs/semantic-model.md | 65 ++++- .../semantic/deploy_knowledge_catalog.ts | 199 +------------ .../mdcode/src/libts/semantic/kc_converter.ts | 267 ++++++++++++++++++ .../src/libts/semantic/knowledge_catalog.ts | 263 +---------------- .../{serialize.ts => osi_converter.ts} | 19 +- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 182 ++++++++++++ toolbox/mdcode/src/tool/commands.ts | 5 +- .../sales_bq_graph_target.osi.golden.yaml | 29 ++ .../sales_bq_graph_target.pull.golden.yaml | 25 ++ .../star_orders_customer.osi.golden.yaml | 86 ++++++ .../star_orders_customer.pull.golden.yaml | 61 ++++ .../fixtures/tpcds_date_edge.osi.golden.yaml | 218 ++++++++++++++ .../fixtures/tpcds_date_edge.pull.golden.yaml | 147 ++++++++++ ...alog.read.test.ts => kc_converter.test.ts} | 63 ++++- ...erialize.test.ts => osi_converter.test.ts} | 52 +++- ...e_catalog.pull.test.ts => pull_kc.test.ts} | 6 +- 16 files changed, 1214 insertions(+), 473 deletions(-) create mode 100644 toolbox/mdcode/src/libts/semantic/kc_converter.ts rename toolbox/mdcode/src/libts/semantic/{serialize.ts => osi_converter.ts} (93%) create mode 100644 toolbox/mdcode/src/libts/semantic/pull_kc.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml rename toolbox/mdcode/tests/libts/semantic/{knowledge_catalog.read.test.ts => kc_converter.test.ts} (73%) rename toolbox/mdcode/tests/libts/semantic/{serialize.test.ts => osi_converter.test.ts} (81%) rename toolbox/mdcode/tests/libts/semantic/{deploy_knowledge_catalog.pull.test.ts => pull_kc.test.ts} (98%) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 19d192ba..7d3c28a2 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -13,8 +13,9 @@ model to two destinations at once: Both are generated from the same source document — you never author them separately, and a single `push` keeps them in sync. -This guide covers authoring, deploying, and updating a model. For the Ossie -document format itself, see [ossie.apache.org](https://ossie.apache.org/). +This guide covers authoring, deploying, pulling back, and updating a model. +For the Ossie document format itself, see +[ossie.apache.org](https://ossie.apache.org/). ## Prerequisites @@ -220,3 +221,63 @@ Every push prints one line per destination summarizing what it did. For a Deployed 1 BigQuery Graph(s). Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. ``` + +## Pull + +`kcmd pull` is the inverse of push's Knowledge Catalog leg: it reads the +`semantic-*` entries back from the catalog and reconstructs local model +documents at `catalog/EntryGroups//.yaml`. Use it to +recover a workspace from a catalog someone else deployed, or to see what the +catalog actually holds. + +```bash +kcmd pull +``` + +Pull reads only from Knowledge Catalog (never BigQuery). Its coordinates come +from the same scope you authored under (`..`). + +| Flag | Effect | +|------|--------| +| `--dry-run` | Reconstruct from the catalog and report what would be written, but write no files. | +| `--model ` | Pull a single model by name; other models in the entry group are left alone. | + +One entry group can hold **many models** — each `semantic-model` entry is a +separate anchor, and pull reconstructs one document per anchor. `--model` +narrows both the fetch and the write to a single anchor. + +Pull writes with the same last-write-wins policy as the core pull: a model that +already exists locally is overwritten in place, and a local-only document (one +with no matching catalog entry) is left untouched — pull never deletes. + +> **Note — pull recovers the catalog, not your authored document.** The catalog +> stores only what push wrote to it (see the note under [What gets created in +> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)), so a pulled +> document comes back without the content the catalog never held: entity keys, +> `ai_context`, field labels, the original vendor SQL (`importedExpression`), +> and relationships (the graph edges live in the BigQuery property graph, not +> the catalog). A field's *role* survives as a bare `dimension: {}` marker, but +> its detail (`is_time`, and so on) does not. Keep your authored document as the +> source of truth; treat a pulled document as a faithful copy of the catalog +> metadata, not of the original model. + +## Permissions + +`push` needs access to whichever destinations you deploy to. + +**BigQuery** — for `--target bq` or `all`, and for the validation pre-flight: + +* `bigquery.jobs.create` in the deployment-target project — to run the deploy's + `CREATE OR REPLACE PROPERTY GRAPH` and the validation dry-run query +* read access on each entity's source table, so the dry-run can resolve it +* `bigquery.datasets.get` on the target dataset (region detection; optional — + push degrades gracefully without it) + +**Knowledge Catalog / Dataplex** — for `--target kc` or `all`: + +* `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group +* `dataplex.entryGroups.useSchemaJoinEntryLink` and + `dataplex.entryGroups.useSchemaJoinAspect` when the model has relationships + +`kcmd pull` needs read access to the same entry group instead — to list its +entries and fetch each `semantic-*` entry with its aspects. diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 0bf69527..5676ec86 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -38,8 +38,7 @@ import {ApiResult} from '../gcp/api'; import * as context from '../gcp/context'; import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; -import {SemanticModel} from './ir'; -import {generateCatalogResources, idOf, KcResources, modelsFromCatalogResources} from './knowledge_catalog'; +import {generateCatalogResources, KcResources} from './knowledge_catalog'; import {LoadedModel} from './loader'; @@ -121,10 +120,6 @@ interface Counts { type EmittedModel = {model: string; resources: KcResources}; -// Upper bound on concurrent entry / entry-link writes within one model's wave -// (mirrors HYDRATE_CONCURRENCY on the pull side). -const WRITE_CONCURRENCY = 8; - // entries.create propagation retry: a just-created entry group can briefly 404. const ENTRY_CREATE_TRIES = 3; const ENTRY_CREATE_RETRY_MS = 3000; @@ -571,8 +566,7 @@ async function createEntries( const isMetric = (e: Entry) => (e.entryType ?? '').endsWith('/semantic-metric'); for (const wave of [children.filter(e => !isMetric(e)), children.filter(isMetric)]) { - const res = await mapConcurrent( - wave, WRITE_CONCURRENCY, e => writeEntry(cat, opts, e)); + const res = await Promise.all(wave.map(e => writeEntry(cat, opts, e))); const firstErr = res.find(r => r.error); if (firstErr) return {created, updated, error: firstErr.error}; for (const r of res) { @@ -599,8 +593,7 @@ async function createEntryLinks( cat: CatalogClient, opts: KcDeployOptions, links: EntryLink[]): Promise { if (!links.length) return {linked: 0}; - const res = await mapConcurrent( - links, WRITE_CONCURRENCY, l => writeEntryLink(cat, opts, l)); + const res = await Promise.all(links.map(l => writeEntryLink(cat, opts, l))); const firstErr = res.find(r => r.error); if (firstErr) return {linked: 0, error: firstErr.error}; return {linked: links.length}; @@ -693,6 +686,11 @@ function planSummary( } +// The id segment of a full entry/entryType resource name (after the last '/'). +function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + function isOk(res: {status: number}): boolean { return res.status === 200; } @@ -717,184 +715,3 @@ function isPropagating(res: {message?: string}): boolean { function errText(res: {status: number; message?: string}): string { return res.message?.trim() || `HTTP ${res.status}`; } - - -// --------------------------------------------------------------------------- -// Pull: Knowledge Catalog -> Semantic Model IR. -// -// The read counterpart of deployKnowledgeCatalog and the inverse of push. -// Unlike a write, a pull needs no server-side type provisioning -- only that -// the `semantic-*` entries exist. It enumerates the entry group, keeps the -// semantic entries, hydrates each one's aspect data (a BASIC list omits aspect -// data, so each entry is re-fetched with its aspect types -- an entity needs -// BOTH its `semantic-entity` aspect and the built-in `schema` aspect), and -// hands the hydrated entries to the pure reader (modelsFromCatalogResources). -// --------------------------------------------------------------------------- - -export interface KcPullOptions { - project: string; - location: string; - entryGroup: string; - model?: string; // limit to a single model by name (default: all) -} - -export interface KcPullResult { - models: SemanticModel[]; - warnings: string[]; -} - -// Upper bound on in-flight aspect-hydration fetches during a pull. -const HYDRATE_CONCURRENCY = 8; - -// Reads the semantic models back from a Knowledge Catalog entry group. Emits no -// console output; warnings (skipped entries, no match for --model, reader -// warnings) are returned for the caller to print. -export async function pullKnowledgeCatalog( - cat: CatalogClient, opts: KcPullOptions): Promise { - const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; - const warnings: string[] = []; - - // Enumerate the group (paging is inherently sequential) and pick the semantic - // entries, then hydrate their aspects concurrently: a BASIC list omits aspect - // data, so each entry needs its own lookupEntry, and those fetches are - // independent. The pool preserves input order so warnings stay deterministic. - const targets: {entry: Entry; aspectTypes: string[]}[] = []; - for await (const entry of cat.listEntries( - opts.project, opts.location, opts.entryGroup)) { - const aspectTypes = semanticAspectTypes(entry.entryType); - if (aspectTypes) targets.push({entry, aspectTypes}); - // else: not part of a semantic model; ignore it. - } - - // When scoped to one model, hydrate only that model's entries -- its anchor - // (matched by name) plus the children pointing at it. A list already carries - // entrySource + parentEntry, so this avoids fetching every other model's - // aspects. No match short-circuits with just the not-found warning. - let scoped = targets; - if (opts.model) { - scoped = scopeToModel(targets, opts.model); - if (!scoped.length) { - return { - models: [], - warnings: [ - `no semantic model named '${opts.model}' found in ${destination}` - ], - }; - } - } - - const fetched = await mapConcurrent( - scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { - const res = await cat.lookupEntry( - opts.project, opts.location, entry.name, aspectTypes); - if (res.status !== 200 || !res.result) { - return { - warning: `failed to fetch entry '${entry.name}' (status ${ - res.status}); skipped` - }; - } - return {entry: res.result}; - }); - - const hydrated: Entry[] = []; - for (const r of fetched) { - if (r.entry) - hydrated.push(r.entry); - else if (r.warning) - warnings.push(r.warning); - } - - const read = modelsFromCatalogResources(hydrated); - warnings.push(...read.warnings); - - // Defense in depth: keep only the requested model even if the reader surfaced - // another anchor (e.g. a child whose parentEntry pointed outside the scope). - let models = read.models; - if (opts.model) { - models = models.filter(m => m.name === opts.model); - if (!models.length) { - warnings.push( - `no semantic model named '${opts.model}' found in ${destination}`); - } - } - - return {models, warnings: [...new Set(warnings)]}; -} - - -// The aspect type resource names to hydrate for a semantic entry, derived from -// its entryType (the aspect types are the parallel resources in the same -// project/location). An entity carries two aspects: its `semantic-entity` -// aspect and the built-in `schema` aspect that holds its fields. Returns -// undefined for entries that are not part of a semantic model. -function semanticAspectTypes(entryType: string): string[]|undefined { - const marker = '/entryTypes/'; - const idx = entryType?.indexOf(marker) ?? -1; - if (idx < 0) return undefined; - const typeBase = entryType.slice(0, idx); - const t = entryType.slice(idx + marker.length); - const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; - switch (t) { - case 'semantic-model': - return [aspectType('semantic-model')]; - case 'semantic-entity': - return [aspectType('semantic-entity'), aspectType('schema')]; - case 'semantic-metric': - return [aspectType('semantic-metric')]; - default: - return undefined; - } -} - - -// Restricts hydration targets to a single model: the semantic-model anchor -// whose name (entrySource.displayName, else the entry id) matches `model`, plus -// every child entry whose parentEntry is that anchor. Uses only list-level -// fields (no aspect data), so it runs before hydration and avoids fetching -// unrelated models' aspects. Returns [] when no anchor matches. -function scopeToModel( - targets: {entry: Entry; aspectTypes: string[]}[], - model: string): {entry: Entry; aspectTypes: string[]}[] { - const isAnchor = (t: {entry: Entry}) => - !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); - const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); - const matchedAnchorNames = new Set( - targets.filter(isAnchor) - .filter( - t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === - model) - .map(t => t.entry.name)); - if (!matchedAnchorNames.size) return []; - // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds - // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a - // project-id normalization mismatch) still belongs to it. Without this a - // scoped pull would drop children a full pull keeps. - const soleAnchor = - allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; - const soleMatched = - soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); - return targets.filter( - t => matchedAnchorNames.has(t.entry.name) || - matchedAnchorNames.has(t.entry.parentEntry ?? '') || - (soleMatched && !isAnchor(t) && - !allAnchorNames.has(t.entry.parentEntry ?? ''))); -} - - -// Maps `items` through `fn` with at most `limit` calls in flight, returning -// results in input order (so downstream ordering stays deterministic). -async function mapConcurrent( - items: T[], limit: number, fn: (item: T) => Promise): Promise { - const results: R[] = new Array(items.length); - let next = 0; - async function worker(): Promise { - while (next < items.length) { - const i = next++; - results[i] = await fn(items[i]); - } - } - const workers = - Array.from({length: Math.min(limit, items.length)}, () => worker()); - await Promise.all(workers); - return results; -} diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts new file mode 100644 index 00000000..233f7b5e --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -0,0 +1,267 @@ +// Knowledge Catalog <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// READ direction -- Knowledge Catalog entries -> IR (`modelsFromCatalogResources` +// and its helpers). The WRITE direction (IR -> KC entries, +// `generateCatalogResources`) still lives in `knowledge_catalog.ts` and moves +// here once PR4 (the KC push line) merges, at which point this becomes the full +// two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is +// the KC emit code?" -> `knowledge_catalog.ts`. +// +// The reader is the inverse of `generateCatalogResources`: it reconstructs the +// IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). +// `semantic-entity` / `semantic-metric` entries are grouped under their +// `semantic-model` anchor via `parentEntry`; entries of other types are ignored. +// Resources are matched by type-name SUFFIX, so a reader need not know which +// system-type project/location the emitter used. +// +// Fidelity is bounded by what the emitter persisted, so this read is the inverse +// of the WRITE, not of the authored document. It recovers names, descriptions, +// data sources, field datatypes (via the schema aspect) and DIMENSION roles, +// field/metric expressions, and each metric's attach entity (re-derived from its +// expression, as the loader does). It cannot recover what the emitter does not +// write: entity keys/unique keys, `ai_context`, field labels, `importedDialect`, +// `custom_extensions`, and relationships (the graph edges live in the BigQuery +// property graph, not the catalog). + +import type {Entry} from '../gcp/dataplex'; +import {DataType, Entity, Field, Metric, SemanticModel} from './ir'; +import {referencedEntityNames} from './sql_expr_utils'; + +export interface ReadResult { + models: SemanticModel[]; + warnings: string[]; +} + +/** + * Reconstructs the Semantic Model IR from Knowledge Catalog entries. + * + * Returns one model per `semantic-model` anchor plus any warnings (no anchor, + * an orphaned child, an entry missing its aspect data). Entries must already be + * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a + * BASIC list omits aspect data, so the puller re-fetches each entry first. + */ +export function modelsFromCatalogResources(entries: Entry[]): ReadResult { + const warnings: string[] = []; + + const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); + const entityEntries = + entries.filter(e => semanticType(e) === 'semantic-entity'); + const metricEntries = + entries.filter(e => semanticType(e) === 'semantic-metric'); + + if (!anchors.length) { + warnings.push('no semantic-model entry found; nothing to reconstruct'); + return {models: [], warnings: [...new Set(warnings)]}; + } + + // A child belongs to its anchor by parentEntry. When there is exactly one + // anchor, children whose parentEntry does not resolve (e.g. a project-id + // normalization mismatch) are still attached to it rather than dropped. + const anchorNames = new Set(anchors.map(a => a.name)); + const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; + const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( + e => e.parentEntry === anchorName || + (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + + const models = anchors.map(anchor => { + const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + + const entities = childrenOf(anchor.name, entityEntries) + .map(e => readEntity(e, warnings)); + const entityNames = entities.map(e => e.name); + const metrics = childrenOf(anchor.name, metricEntries) + .map(e => readMetric(e, entityNames, warnings)); + + // Relationships are not published to the catalog (see the file header), so + // a reconstructed model always has an empty edge set. + const model: SemanticModel = {name, entities, relationships: [], metrics}; + const description = anchor.entrySource?.description; + if (description !== undefined) model.description = description; + return model; + }); + + // Flag children that resolved to no anchor at all (only possible with + // multiple anchors, where the sole-anchor fallback does not apply). + if (!soleAnchor) { + for (const child of [...entityEntries, ...metricEntries]) { + if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { + warnings.push(`entry '${ + child.name}' has no resolvable parent semantic-model; omitted`); + } + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// Reconstructs an entity from its `semantic-entity` aspect (the backing source) +// and the built-in `schema` aspect (its fields). Keys are not persisted by the +// emitter and so come back empty. +function readEntity(entry: Entry, warnings: string[]): Entity { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const semantic = aspectData(entry, 'semantic-entity'); + const schema = aspectData(entry, 'schema'); + if (!Object.keys(semantic).length) { + warnings.push(`entity '${ + name}': no semantic-entity aspect data (fetch with the aspect type)`); + } + + const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); + if (!dataSource) { + warnings.push( + `entity '${name}': no backing data source in the semantic-entity ` + + `aspect; 'source' will be empty and the entity may not load`); + } + const entity: Entity = { + name, + dataSource, + keys: [], // not persisted by the emitter; unrecoverable on read + fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + }; + const description = entry.entrySource?.description; + if (description !== undefined) entity.description = description; + return entity; +} + + +// Reconstructs a field from one `schema` aspect field record, inverting +// schemaAspectData: the datatype from dataType/metadataType, expressions from +// the nested `semantics` block, and the DIMENSION role back to a dimension +// marker. +function readField(fd: any, entityName: string, warnings: string[]): Field { + const name = fd?.name; + if (name === undefined || name === '') { + warnings.push( + `entity '${entityName}': a schema field is missing its name; the ` + + `field may not load`); + } + const field: Field = {name}; + const sem = fd?.semantics ?? {}; + if (sem.expression !== undefined) field.expression = sem.expression; + const type = irDataType(fd?.dataType, fd?.metadataType); + if (type !== undefined) field.type = type; + if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.description !== undefined) field.description = fd.description; + return field; +} + + +// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` +// is re-derived from the expression (as the loader does) rather than read from +// the aspect, so it stays consistent with the reconstructed entity set. +function readMetric( + entry: Entry, entityNames: string[], warnings: string[]): Metric { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + if (data.expression === undefined) { + warnings.push(`metric '${name}': no expression in semantic-metric aspect`); + } + + const metric: Metric = {name}; + if (data.expression !== undefined) metric.expression = data.expression; + const exprForRefs = data.expression ?? ''; + const referenced = referencedEntityNames(exprForRefs, entityNames); + if (referenced.length === 1) { + metric.entity = referenced[0]; + } else if (exprForRefs && !referenced.length) { + // Parity with the loader's convertMetric: an expression that qualifies no + // known entity is flagged as potentially unplaceable downstream. + warnings.push( + `metric '${name}': expression references no known entity; it may not ` + + `be placeable downstream`); + } + // The emitter writes a required dataType, defaulting a typeless metric to + // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric + // authored without a datatype round-trips as an explicit Decimal rather than + // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) + const type = irDataType(data.dataType, undefined); + if (type !== undefined) metric.type = type; + const description = entry.entrySource?.description; + if (description !== undefined) metric.description = description; + return metric; +} + + +// The inverse of columnDataType/columnMetadataType: maps the schema aspect's +// dataType (disambiguated by metadataType only for the STRING family) back to +// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read +// as un-typed (undefined) -- the loader's default -- since the emitter cannot +// distinguish an authored `String` from an un-typed field (both emit STRING). +function irDataType(dataType: string|undefined, metadataType: string|undefined): + DataType|undefined { + switch (dataType) { + case 'INT64': + return 'Integer'; + case 'NUMERIC': + return 'Decimal'; + case 'FLOAT64': + return 'Float'; + case 'BOOL': + return 'Boolean'; + case 'DATE': + return 'Date'; + case 'TIME': + return 'Time'; + case 'DATETIME': + return 'DateTime'; + case 'TIMESTAMP': + return 'DateTimeTz'; + case 'STRING': + return metadataType === 'OTHER' ? 'Opaque' : undefined; + default: + return undefined; + } +} + + +// The inverse of resourcePath: a BigQuery linked-resource URI becomes the +// canonical `project.dataset.table` string; anything else (a verbatim query or +// a passthrough reference) is returned unchanged. +function dataSourceFromResource(resource: string|undefined): string { + const value = (resource ?? '').trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + + +// The bare `semantic-*` type of an entry, matched by entryType suffix so the +// system-type project/location need not be known. Returns undefined for entries +// that are not part of a semantic model. +function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| + 'semantic-metric'|undefined { + for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as + const) { + if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; + } + return undefined; +} + + +// The `data` payload of an entry's aspect of the given bare type, matched by +// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` +// suffix (robust to whichever system-type project/location the emitter used). +// Returns an empty object when the aspect is absent. +function aspectData(entry: Entry, type: string): Record { + const aspects = entry.aspects ?? {}; + for (const [key, aspect] of Object.entries(aspects)) { + if (key.endsWith(`.${type}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + + +// The id segment of a full entry resource name (after the last '/'). +export function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index 5e5860c9..a3051b22 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -45,8 +45,7 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {bigQueryGraphTargets} from './deploy_bigquery'; -import {DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; -import {referencedEntityNames} from './sql_expr_utils'; +import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -540,263 +539,3 @@ function linkSlug(s: string): string { function unquote(part: string): string { return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); } - - -// --------------------------------------------------------------------------- -// Reader: Knowledge Catalog entries -> Semantic Model IR. -// -// The inverse of generateCatalogResources: it reconstructs the IR from the -// entries a pull hydrated (see deploy_knowledge_catalog.pullKnowledgeCatalog). -// `semantic-entity` / `semantic-metric` entries are grouped under their -// `semantic-model` anchor via `parentEntry`; entries of other types are -// ignored. Resources are matched by type-name SUFFIX, so a reader need not know -// which system-type project/location the emitter used. -// -// Fidelity is bounded by what the emitter persisted, so this read is the -// inverse of the WRITE, not of the authored document. It recovers names, -// descriptions, data sources, field datatypes (via the schema aspect) and -// DIMENSION roles, field/metric expressions, and each -// metric's attach entity (re-derived from its expression, as the loader does). -// It cannot recover what the emitter does not write: entity keys/unique keys, -// `ai_context`, field labels, `importedDialect`, `custom_extensions`, and -// relationships (the graph edges live in the BigQuery property graph, not the -// catalog). -// --------------------------------------------------------------------------- - -export interface ReadResult { - models: SemanticModel[]; - warnings: string[]; -} - -/** - * Reconstructs the Semantic Model IR from Knowledge Catalog entries. - * - * Returns one model per `semantic-model` anchor plus any warnings (no anchor, - * an orphaned child, an entry missing its aspect data). Entries must already be - * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a - * BASIC list omits aspect data, so the puller re-fetches each entry first. - */ -export function modelsFromCatalogResources(entries: Entry[]): ReadResult { - const warnings: string[] = []; - - const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); - const entityEntries = - entries.filter(e => semanticType(e) === 'semantic-entity'); - const metricEntries = - entries.filter(e => semanticType(e) === 'semantic-metric'); - - if (!anchors.length) { - warnings.push('no semantic-model entry found; nothing to reconstruct'); - return {models: [], warnings: [...new Set(warnings)]}; - } - - // A child belongs to its anchor by parentEntry. When there is exactly one - // anchor, children whose parentEntry does not resolve (e.g. a project-id - // normalization mismatch) are still attached to it rather than dropped. - const anchorNames = new Set(anchors.map(a => a.name)); - const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; - const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( - e => e.parentEntry === anchorName || - (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); - - const models = anchors.map(anchor => { - const name = anchor.entrySource?.displayName ?? idOf(anchor.name); - - const entities = childrenOf(anchor.name, entityEntries) - .map(e => readEntity(e, warnings)); - const entityNames = entities.map(e => e.name); - const metrics = childrenOf(anchor.name, metricEntries) - .map(e => readMetric(e, entityNames, warnings)); - - // Relationships are not published to the catalog (see the file header), so - // a reconstructed model always has an empty edge set. - const model: SemanticModel = {name, entities, relationships: [], metrics}; - const description = anchor.entrySource?.description; - if (description !== undefined) model.description = description; - return model; - }); - - // Flag children that resolved to no anchor at all (only possible with - // multiple anchors, where the sole-anchor fallback does not apply). - if (!soleAnchor) { - for (const child of [...entityEntries, ...metricEntries]) { - if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { - warnings.push(`entry '${ - child.name}' has no resolvable parent semantic-model; omitted`); - } - } - } - - return {models, warnings: [...new Set(warnings)]}; -} - - -// Reconstructs an entity from its `semantic-entity` aspect (the backing source) -// and the built-in `schema` aspect (its fields). Keys are not persisted by the -// emitter and so come back empty. -function readEntity(entry: Entry, warnings: string[]): Entity { - const name = entry.entrySource?.displayName ?? idOf(entry.name); - const semantic = aspectData(entry, 'semantic-entity'); - const schema = aspectData(entry, 'schema'); - if (!Object.keys(semantic).length) { - warnings.push(`entity '${ - name}': no semantic-entity aspect data (fetch with the aspect type)`); - } - - const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); - if (!dataSource) { - warnings.push( - `entity '${name}': no backing data source in the semantic-entity ` + - `aspect; 'source' will be empty and the entity may not load`); - } - const entity: Entity = { - name, - dataSource, - keys: [], // not persisted by the emitter; unrecoverable on read - fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), - }; - const description = entry.entrySource?.description; - if (description !== undefined) entity.description = description; - return entity; -} - - -// Reconstructs a field from one `schema` aspect field record, inverting -// schemaAspectData: the datatype from dataType/metadataType, expressions from -// the nested `semantics` block, and the DIMENSION role back to a dimension -// marker. -function readField(fd: any, entityName: string, warnings: string[]): Field { - const name = fd?.name; - if (name === undefined || name === '') { - warnings.push( - `entity '${entityName}': a schema field is missing its name; the ` + - `field may not load`); - } - const field: Field = {name}; - const sem = fd?.semantics ?? {}; - if (sem.expression !== undefined) field.expression = sem.expression; - const type = irDataType(fd?.dataType, fd?.metadataType); - if (type !== undefined) field.type = type; - if (sem.role === 'DIMENSION') field.dimension = {}; - if (fd?.description !== undefined) field.description = fd.description; - return field; -} - - -// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` -// is re-derived from the expression (as the loader does) rather than read from -// the aspect, so it stays consistent with the reconstructed entity set. -function readMetric( - entry: Entry, entityNames: string[], warnings: string[]): Metric { - const name = entry.entrySource?.displayName ?? idOf(entry.name); - const data = aspectData(entry, 'semantic-metric'); - if (data.expression === undefined) { - warnings.push(`metric '${name}': no expression in semantic-metric aspect`); - } - - const metric: Metric = {name}; - if (data.expression !== undefined) metric.expression = data.expression; - const exprForRefs = data.expression ?? ''; - const referenced = referencedEntityNames(exprForRefs, entityNames); - if (referenced.length === 1) { - metric.entity = referenced[0]; - } else if (exprForRefs && !referenced.length) { - // Parity with the loader's convertMetric: an expression that qualifies no - // known entity is flagged as potentially unplaceable downstream. - warnings.push( - `metric '${name}': expression references no known entity; it may not ` + - `be placeable downstream`); - } - // The emitter writes a required dataType, defaulting a typeless metric to - // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric - // authored without a datatype round-trips as an explicit Decimal rather than - // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) - const type = irDataType(data.dataType, undefined); - if (type !== undefined) metric.type = type; - const description = entry.entrySource?.description; - if (description !== undefined) metric.description = description; - return metric; -} - - -// The inverse of columnDataType/columnMetadataType: maps the schema aspect's -// dataType (disambiguated by metadataType only for the STRING family) back to -// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read -// as un-typed (undefined) -- the loader's default -- since the emitter cannot -// distinguish an authored `String` from an un-typed field (both emit STRING). -function irDataType(dataType: string|undefined, metadataType: string|undefined): - DataType|undefined { - switch (dataType) { - case 'INT64': - return 'Integer'; - case 'NUMERIC': - return 'Decimal'; - case 'FLOAT64': - return 'Float'; - case 'BOOL': - return 'Boolean'; - case 'DATE': - return 'Date'; - case 'TIME': - return 'Time'; - case 'DATETIME': - return 'DateTime'; - case 'TIMESTAMP': - return 'DateTimeTz'; - case 'STRING': - return metadataType === 'OTHER' ? 'Opaque' : undefined; - default: - return undefined; - } -} - - -// The inverse of resourcePath: a BigQuery linked-resource URI becomes the -// canonical `project.dataset.table` string; anything else (a verbatim query or -// a passthrough reference) is returned unchanged. -function dataSourceFromResource(resource: string|undefined): string { - const value = (resource ?? '').trim(); - const m = value.match( - /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); - return m ? `${m[1]}.${m[2]}.${m[3]}` : value; -} - - -// The bare `semantic-*` type of an entry, matched by entryType suffix so the -// system-type project/location need not be known. Returns undefined for entries -// that are not part of a semantic model. -function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| - 'semantic-metric'|undefined { - for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as - const) { - if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; - } - return undefined; -} - - -// The `data` payload of an entry's aspect of the given bare type, matched by -// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` -// suffix (robust to whichever system-type project/location the emitter used). -// Returns an empty object when the aspect is absent. -function aspectData(entry: Entry, type: string): Record { - const aspects = entry.aspects ?? {}; - for (const [key, aspect] of Object.entries(aspects)) { - if (key.endsWith(`.${type}`) || - aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { - return aspect.data ?? {}; - } - } - return {}; -} - - -// The id segment of a full entry resource name (after the last '/'). -export function idOf(name: string): string { - return name.split('/').pop() ?? name; -} - - -function asArray(value: any): any[] { - return Array.isArray(value) ? value : []; -} diff --git a/toolbox/mdcode/src/libts/semantic/serialize.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts similarity index 93% rename from toolbox/mdcode/src/libts/semantic/serialize.ts rename to toolbox/mdcode/src/libts/semantic/osi_converter.ts index dd215100..61c4a99b 100644 --- a/toolbox/mdcode/src/libts/semantic/serialize.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -1,9 +1,16 @@ -// Serializes the Semantic Model IR (./ir) back to the open AI-first semantics -// format (YAML). This is the inverse of `loader.ts`: `loader` reads authored -// YAML into the IR; this module writes the IR out as a YAML document the loader -// can read back. It is the local-workspace sink for `pull` (Knowledge Catalog -// -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR -> a -// destination. +// OSI (open, AI-first semantics) <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// WRITE direction -- IR -> OSI YAML (`serializeModel`). The READ direction +// (OSI YAML -> IR) still lives in `loader.ts` and moves here once PR4 (the +// KC push line) merges, at which point this becomes the full two-way +// converter. Until then, "where is the OSI parser?" -> `loader.ts`. +// +// Serializing is the inverse of `loader.ts`: `loader` reads authored YAML +// into the IR; this module writes the IR out as a YAML document the loader +// can read back. It is the local-workspace sink for `pull` (Knowledge +// Catalog -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR +// -> a destination. // // Fidelity is at the IR level, not byte-for-byte with a hand-authored file. The // loader normalizes several authoring conveniences into the IR at load time, so diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts new file mode 100644 index 00000000..f03edc3f --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -0,0 +1,182 @@ +// Pull: fetch a semantic model from a live Knowledge Catalog into the IR. +// +// The read-direction counterpart of `deployKnowledgeCatalog` (push, in +// `deploy_knowledge_catalog.ts`). Unlike a write, a pull needs no server-side +// type provisioning -- only that the `semantic-*` entries exist. It enumerates +// the entry group, keeps the semantic entries, hydrates each one's aspect data +// (a BASIC list omits aspect data, so each entry is re-fetched with its aspect +// types -- an entity needs BOTH its `semantic-entity` aspect and the built-in +// `schema` aspect), and hands the hydrated entries to the pure reader +// (`kc_converter.modelsFromCatalogResources`). + +import {CatalogClient, Entry} from '../gcp/dataplex'; +import {SemanticModel} from './ir'; +import {idOf, modelsFromCatalogResources} from './kc_converter'; + +export interface KcPullOptions { + project: string; + location: string; + entryGroup: string; + model?: string; // limit to a single model by name (default: all) +} + +export interface KcPullResult { + models: SemanticModel[]; + warnings: string[]; +} + +// Upper bound on in-flight aspect-hydration fetches during a pull. +const HYDRATE_CONCURRENCY = 8; + +// Reads the semantic models back from a Knowledge Catalog entry group. Emits no +// console output; warnings (skipped entries, no match for --model, reader +// warnings) are returned for the caller to print. +export async function pullKnowledgeCatalog( + cat: CatalogClient, opts: KcPullOptions): Promise { + const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const warnings: string[] = []; + + // Enumerate the group (paging is inherently sequential) and pick the semantic + // entries, then hydrate their aspects concurrently: a BASIC list omits aspect + // data, so each entry needs its own lookupEntry, and those fetches are + // independent. The pool preserves input order so warnings stay deterministic. + const targets: {entry: Entry; aspectTypes: string[]}[] = []; + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const aspectTypes = semanticAspectTypes(entry.entryType); + if (aspectTypes) targets.push({entry, aspectTypes}); + // else: not part of a semantic model; ignore it. + } + + // When scoped to one model, hydrate only that model's entries -- its anchor + // (matched by name) plus the children pointing at it. A list already carries + // entrySource + parentEntry, so this avoids fetching every other model's + // aspects. No match short-circuits with just the not-found warning. + let scoped = targets; + if (opts.model) { + scoped = scopeToModel(targets, opts.model); + if (!scoped.length) { + return { + models: [], + warnings: [ + `no semantic model named '${opts.model}' found in ${destination}` + ], + }; + } + } + + const fetched = await mapConcurrent( + scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry '${entry.name}' (status ${ + res.status}); skipped` + }; + } + return {entry: res.result}; + }); + + const hydrated: Entry[] = []; + for (const r of fetched) { + if (r.entry) + hydrated.push(r.entry); + else if (r.warning) + warnings.push(r.warning); + } + + const read = modelsFromCatalogResources(hydrated); + warnings.push(...read.warnings); + + // Defense in depth: keep only the requested model even if the reader surfaced + // another anchor (e.g. a child whose parentEntry pointed outside the scope). + let models = read.models; + if (opts.model) { + models = models.filter(m => m.name === opts.model); + if (!models.length) { + warnings.push( + `no semantic model named '${opts.model}' found in ${destination}`); + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// The aspect type resource names to hydrate for a semantic entry, derived from +// its entryType (the aspect types are the parallel resources in the same +// project/location). An entity carries two aspects: its `semantic-entity` +// aspect and the built-in `schema` aspect that holds its fields. Returns +// undefined for entries that are not part of a semantic model. +function semanticAspectTypes(entryType: string): string[]|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + const typeBase = entryType.slice(0, idx); + const t = entryType.slice(idx + marker.length); + const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + switch (t) { + case 'semantic-model': + return [aspectType('semantic-model')]; + case 'semantic-entity': + return [aspectType('semantic-entity'), aspectType('schema')]; + case 'semantic-metric': + return [aspectType('semantic-metric')]; + default: + return undefined; + } +} + + +// Restricts hydration targets to a single model: the semantic-model anchor +// whose name (entrySource.displayName, else the entry id) matches `model`, plus +// every child entry whose parentEntry is that anchor. Uses only list-level +// fields (no aspect data), so it runs before hydration and avoids fetching +// unrelated models' aspects. Returns [] when no anchor matches. +function scopeToModel( + targets: {entry: Entry; aspectTypes: string[]}[], + model: string): {entry: Entry; aspectTypes: string[]}[] { + const isAnchor = (t: {entry: Entry}) => + !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); + const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = new Set( + targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === + model) + .map(t => t.entry.name)); + if (!matchedAnchorNames.size) return []; + // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds + // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a + // project-id normalization mismatch) still belongs to it. Without this a + // scoped pull would drop children a full pull keeps. + const soleAnchor = + allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; + const soleMatched = + soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); + return targets.filter( + t => matchedAnchorNames.has(t.entry.name) || + matchedAnchorNames.has(t.entry.parentEntry ?? '') || + (soleMatched && !isAnchor(t) && + !allAnchorNames.has(t.entry.parentEntry ?? ''))); +} + + +// Maps `items` through `fn` with at most `limit` calls in flight, returning +// results in input order (so downstream ordering stays deterministic). +async function mapConcurrent( + items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker(): Promise { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + } + const workers = + Array.from({length: Math.min(limit, items.length)}, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 59ce84e1..1c67d76f 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -14,7 +14,8 @@ import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; import {BigQueryClient} from '../libts/gcp/bigquery'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; -import { serializeModel } from '../libts/semantic/serialize'; +import {pullKnowledgeCatalog} from '../libts/semantic/pull_kc'; +import {serializeModel} from '../libts/semantic/osi_converter'; import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; @@ -387,7 +388,7 @@ async function pullSemanticModel( : 'Pulling semantic model from Knowledge Catalog...'); const catalog = new dataplex.CatalogClient(ctx); - const result = await kc.pullKnowledgeCatalog(catalog, { + const result = await pullKnowledgeCatalog(catalog, { project: source.project, location: source.location, entryGroup: source.entryGroup, diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml new file mode 100644 index 00000000..ffdfa798 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml @@ -0,0 +1,29 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: + - name: orders + source: demo.sales.orders + primary_key: + - o_orderkey + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml new file mode 100644 index 00000000..f9e42580 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -0,0 +1,25 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + datasets: + - name: orders + source: demo.sales.orders + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml new file mode 100644 index 00000000..40c22466 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml @@ -0,0 +1,86 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + ai_context: + instructions: Use this model for order analysis. + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' + datasets: + - name: orders + source: samples.tpch.orders + primary_key: + - o_orderkey + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + label: Order Date + dimension: + is_time: true + ai_context: + synonyms: + - order date + - date + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + - name: customer + source: samples.tpch.customer + primary_key: + - c_custkey + fields: + - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + description: Total order revenue + ai_context: + synonyms: + - revenue + - sales + - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml new file mode 100644 index 00000000..bf7cfc8d --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -0,0 +1,61 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + datasets: + - name: orders + source: samples.tpch.orders + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + dimension: {} + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + - name: customer + source: samples.tpch.customer + fields: + - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name + description: Customer name + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + datatype: Decimal + description: Total order revenue + - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) + datatype: Decimal + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml new file mode 100644 index 00000000..27acf1e6 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml @@ -0,0 +1,218 @@ +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + primary_key: + - ss_item_sk + - ss_ticket_number + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_item_sk}" + dimension: + is_time: false + - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_ticket_number}" + dimension: + is_time: false + - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_customer_sk}" + dimension: + is_time: false + - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_store_sk}" + dimension: + is_time: false + - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_quantity}" + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_sales_price}" + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_ext_sales_price}" + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_net_profit}" + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + primary_key: + - c_customer_sk + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: + is_time: false + - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: + is_time: false + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: + is_time: false + description: Customer last name + - name: item + source: tpcds.public.item + primary_key: + - i_item_sk + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: + is_time: false + - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: + is_time: false + - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: + is_time: false + - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + primary_key: + - s_store_sk + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: + is_time: false + - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: + is_time: false + - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: + is_time: false + - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: + is_time: false + - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes + custom_extensions: + - vendor_name: GOODDATA + data: '{"date_dimension": true, "granularities": ["DAY", "WEEK", "MONTH", + "QUARTER", "YEAR"]}' + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml new file mode 100644 index 00000000..7437f8ac --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -0,0 +1,147 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + dimension: {} + - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + dimension: {} + - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + dimension: {} + - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + dimension: {} + - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: {} + - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: {} + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: {} + description: Customer last name + - name: item + source: tpcds.public.item + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: {} + - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: {} + - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: {} + - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: {} + - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: {} + - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: {} + - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: {} + - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts similarity index 73% rename from toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts rename to toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index a8d5e447..d5ecb145 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -1,5 +1,5 @@ -// Behavior specification for the Knowledge Catalog reader -// (modelsFromCatalogResources in src/libts/semantic/knowledge_catalog.ts). +// Behavior specification for the KC converter's read direction +// (modelsFromCatalogResources in src/libts/semantic/kc_converter.ts). // // The reader is the inverse of the emitter (generateCatalogResources). The // central guarantee is an emitter -> reader round trip: emit a model's entries, @@ -12,9 +12,15 @@ // re-derivation, and parent/anchor grouping). import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {generateCatalogResources, modelsFromCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {serializeModel} from '../../../src/libts/semantic/osi_converter'; + +const FIXTURES = path.join(__dirname, 'fixtures'); const OPTS = { project: 'dest', @@ -256,3 +262,54 @@ describe('metric expression referencing no known entity', () => { .toBe(true); }); }); + + +// -- Golden pull: the whole KC entries -> IR -> OSI YAML output. -- +// +// The round trip above proves the reader inverts the emitter in memory; this +// pins the reviewable artifact. For each corpus fixture it reads the committed +// emitter golden (`.knowledge_catalog.golden.json` -- the exact entries +// a push produced) back through the reader and serializes the reconstructed IR +// to `.pull.golden.yaml`. Open that next to the fixture's +// `.osi.golden.yaml` to see, as whole files, what a Knowledge Catalog round trip +// preserves and what it drops (keys, ai_context, labels, relationships). +// +// Regenerate after an intentional reader/serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts +describe('golden pull: each corpus KC golden reconstructs to its exact YAML', + () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + const kcGoldenPath = (fixture: string) => path.join( + FIXTURES, + fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + const pullGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const {models, warnings} = modelsFromCatalogResources(kc.entries); + // Reader warnings ride along as YAML comments so the golden shows + // the full outcome, not just the recovered document. + const header = warnings.length ? + warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : + '# (no warnings)\n'; + const actual = + header + models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = pullGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts similarity index 81% rename from toolbox/mdcode/tests/libts/semantic/serialize.test.ts rename to toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts index 81c4454c..82943898 100644 --- a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts @@ -1,7 +1,7 @@ -// Behavior specification for the semantic-model serializer -// (src/libts/semantic/serialize.ts). +// Behavior specification for the OSI converter's serialize direction +// (serializeModel in src/libts/semantic/osi_converter.ts). // -// serialize.ts is the inverse of loader.ts: IR -> open-format YAML. The +// The serializer is the inverse of loader.ts: IR -> open-format YAML. The // strongest guarantee is a round trip through the loader -- load a fixture to // the IR, serialize it, load the serialized text again, and assert the two IRs // are identical. That pins IR-level fidelity across every feature the loader @@ -17,7 +17,7 @@ import * as yaml from 'yaml'; import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; import {loadModels} from '../../../src/libts/semantic/loader'; -import {modelDocument, serializeModel} from '../../../src/libts/semantic/serialize'; +import {modelDocument, serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -288,3 +288,47 @@ describe('serialize flags loader-invalid reconstructions', () => { expect(labels).toContain('BIGQUERY'); }); }); + + +// -- Golden corpus: the whole IR -> OSI YAML output, reviewable as a file. -- +// +// The round-trip tests above prove IR-level stability but never pin the exact +// YAML text. These goldens capture the full serialized document for a small +// corpus so a reviewer can open a `.yaml` next to its `.osi.golden.yaml` and see +// exactly what the serializer emits. The same corpus backs kc_converter's +// `.pull.golden.yaml`, so diffing the two shows what a Knowledge Catalog round +// trip loses relative to the authored source. +// +// Regenerate after an intentional serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/osi_converter.test.ts +describe('golden OSI document: each corpus fixture serializes to its exact YAML', + () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the KC e2e/pull goldens, so the OSI golden + // and the pull golden are directly comparable. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + const osiGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.osi.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + const models = loadModels(text, LOAD).models; + const actual = models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = osiGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts similarity index 98% rename from toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts rename to toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index e31a2f94..5a09cebd 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -1,5 +1,5 @@ // Tests for the semantic-model Knowledge Catalog pull leg -// (pullKnowledgeCatalog in src/libts/semantic/deploy_knowledge_catalog.ts). +// (pullKnowledgeCatalog in src/libts/semantic/pull_kc.ts). // // pullKnowledgeCatalog is the orchestration around the pure reader: enumerate // the entry group, hydrate each semantic entry's aspect data, and reconstruct @@ -7,7 +7,7 @@ // the fake serves are produced by the real emitter, so this exercises the true // list -> hydrate -> read path end to end (an entity is re-fetched with BOTH // its semantic-entity and schema aspects). The reader's own mapping is covered -// in knowledge_catalog.read.test.ts; the focus here is the fetch SEQUENCE: +// in kc_converter.test.ts; the focus here is the fetch SEQUENCE: // aspect hydration, the --model filter, skipped entries, and ignoring foreign // entries. @@ -15,7 +15,7 @@ import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; import {ApiResult} from '../../../src/libts/gcp/api'; import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; -import {pullKnowledgeCatalog} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; import {SemanticModel} from '../../../src/libts/semantic/ir'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; From da163dc4d4e29cf048346f693622c777fc00c76a Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:29:59 +0000 Subject: [PATCH 03/12] mdcode: add TODO(#278) migration notes to the converter-scaffold files Each new converter/orchestration file now carries an actionable TODO spelling out how the scaffold collapses once #278 merges, so reviewers can see the plan: - osi_converter.ts: fold loader.ts (OSI read) in, delete loader.ts, repoint importers. - kc_converter.ts: fold generateCatalogResources (KC write) in, delete knowledge_catalog.ts, repoint importers, demote the shared idOf to a local. - pull_kc.ts: rename deploy_knowledge_catalog.ts -> push_kc.ts for push_kc/pull_kc symmetry (rename only, no logic moves). --- toolbox/mdcode/src/libts/semantic/kc_converter.ts | 8 ++++++++ toolbox/mdcode/src/libts/semantic/osi_converter.ts | 7 +++++++ toolbox/mdcode/src/libts/semantic/pull_kc.ts | 7 +++++++ 3 files changed, 22 insertions(+) diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 233f7b5e..25a67c0b 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -8,6 +8,14 @@ // two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is // the KC emit code?" -> `knowledge_catalog.ts`. // +// TODO(#278): fold the KC WRITE direction in and drop the scaffold. Once +// #278 merges, move `generateCatalogResources` (and the `KcResources` type +// + emit helpers) out of `knowledge_catalog.ts` into this file, delete +// `knowledge_catalog.ts`, and repoint its importers (`deploy_knowledge_catalog.ts` +// and the emitter tests). `idOf` -- shared by both directions and re-exported +// from here only for `pull_kc` today -- becomes a plain local. Then this is +// the sole KC<->IR codec and the SCAFFOLD note above comes out. +// // The reader is the inverse of `generateCatalogResources`: it reconstructs the // IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). // `semantic-entity` / `semantic-metric` entries are grouped under their diff --git a/toolbox/mdcode/src/libts/semantic/osi_converter.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts index 61c4a99b..b8b260aa 100644 --- a/toolbox/mdcode/src/libts/semantic/osi_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -6,6 +6,13 @@ // KC push line) merges, at which point this becomes the full two-way // converter. Until then, "where is the OSI parser?" -> `loader.ts`. // +// TODO(#278): fold the OSI READ direction in and drop the scaffold. Once +// #278 merges, move `loadModels` / `fromDocument` / `loadSemanticModels` +// (and their Load* types) out of `loader.ts` into this file, delete +// `loader.ts`, and repoint its importers (`src/tool/commands.ts` and the +// load / validate / bigquery tests). Then this is the sole OSI<->IR codec +// and the SCAFFOLD note above comes out. +// // Serializing is the inverse of `loader.ts`: `loader` reads authored YAML // into the IR; this module writes the IR out as a YAML document the loader // can read back. It is the local-workspace sink for `pull` (Knowledge diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index f03edc3f..aebaa7b5 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -8,6 +8,13 @@ // types -- an entity needs BOTH its `semantic-entity` aspect and the built-in // `schema` aspect), and hands the hydrated entries to the pure reader // (`kc_converter.modelsFromCatalogResources`). +// +// TODO(#278): Layer 2 push/pull symmetry. This is the pull half over the +// pure `kc_converter` codec; the push half (`deployKnowledgeCatalog`) still +// lives in `deploy_knowledge_catalog.ts`. Once #278 merges, rename that +// file to `push_kc.ts` (repointing `src/tool/commands.ts` and its test) so +// the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no +// logic moves. import {CatalogClient, Entry} from '../gcp/dataplex'; import {SemanticModel} from './ir'; From 5b41f6429c5afa2f168c716df81e045d288efd26 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:34:51 +0000 Subject: [PATCH 04/12] mdcode: state push/pull lossiness explicitly in the user guide Reviewers asked the docs to be clear about lossless vs lossy. Both directions are lossy; say so plainly and enumerate exactly what each drops: - Push to BigQuery is lossy: captures the queryable structure (node/edge tables, measures) but not descriptive metadata; non-reducible metrics are skipped. - Push to Knowledge Catalog is lossy: stores a metadata subset (keeps 1:1/1:N as schema-join links) and drops keys, ai_context, labels, vendor SQL, M:N. - Pull is lossy: recovers even less than the catalog holds (no relationships, no deploymentTargets). A push followed by a pull does not return the original file. --- toolbox/mdcode/docs/semantic-model.md | 49 ++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 7d3c28a2..80b7db91 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -138,6 +138,12 @@ statement runs in the right region; without that permission it falls back to BigQuery's own location inference and warns. Nothing runs under `--validate-only`; add `--print` to see the DDL. +Push to BigQuery is **lossy**: the graph captures the queryable structure — +node tables, edge tables, and measures — but not descriptive metadata +(descriptions, `ai_context`, synonyms, labels), and a metric that does not +reduce to a single MEASURE is dropped with a warning. It is a query surface, +not a copy of your model. + ### What gets created in Knowledge Catalog Each element of your model maps to one catalog resource. Every resource type @@ -155,16 +161,19 @@ An entity entry carries its columns in the `schema` aspect (name, data type, and description per field); a `schema-join` link carries the relationship detail — the paired columns and foreign-key direction — in its aspect. -> **Note — the catalog is not a full copy of your model.** By default the SQL -> expressions are **not** written to Knowledge Catalog: the published system-type -> templates do not yet carry a per-field `semantics` block or a -> `semantic-metric.expression` field, so the default push omits them (pass -> `--emit-expressions` to write them once the templates gain the fields). The -> original vendor SQL (`importedExpression` — e.g. the MAQL or Snowflake form a -> metric was imported from) is never written either. All of it stays in your -> authored document and is still used when generating BigQuery SQL. Keep your -> model document as the source of truth: a model reconstructed only from the -> catalog would come back without its SQL. +> **Note — push to Knowledge Catalog is lossy.** The catalog holds metadata, +> not a full copy of your model. It **stores** names, descriptions, data +> sources, field datatypes and roles, and 1:1 / 1:N relationships (as +> `schema-join` links). By default it does **not** store the SQL expressions: +> the published system-type templates do not yet carry a per-field `semantics` +> block or a `semantic-metric.expression` field, so the default push omits them +> (pass `--emit-expressions` to write the canonical GoogleSQL/ANSI expression +> once the templates gain the fields). It never stores entity keys, `ai_context`, +> field labels, the original vendor SQL (`importedExpression` — e.g. the MAQL or +> Snowflake form a metric was imported from), or M:N relationships. Those stay in +> your authored document (and, for the edges, in the BigQuery property graph); the +> vendor SQL and expressions are still used when generating BigQuery SQL. Keep +> your model document as the source of truth. ## Validation @@ -250,16 +259,18 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull recovers the catalog, not your authored document.** The catalog -> stores only what push wrote to it (see the note under [What gets created in -> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)), so a pulled -> document comes back without the content the catalog never held: entity keys, +> **Note — pull is lossy.** It reconstructs a model only from what push wrote +> to the catalog (see the note under [What gets created in Knowledge +> Catalog](#what-gets-created-in-knowledge-catalog)), and recovers even less +> than the catalog holds. A pulled document comes back without entity keys, > `ai_context`, field labels, the original vendor SQL (`importedExpression`), -> and relationships (the graph edges live in the BigQuery property graph, not -> the catalog). A field's *role* survives as a bare `dimension: {}` marker, but -> its detail (`is_time`, and so on) does not. Keep your authored document as the -> source of truth; treat a pulled document as a faithful copy of the catalog -> metadata, not of the original model. +> relationships (even the 1:1 / 1:N `schema-join` links push wrote — the edges +> live in the BigQuery property graph), and the `deploymentTargets` custom +> extension. A field's *role* survives as a bare `dimension: {}` marker, but its +> detail (`is_time`, and so on) does not. **A push followed by a pull does not +> return your original file** — treat a pulled document as a faithful copy of +> the catalog metadata, not of the authored model, and keep the authored +> document as the source of truth. ## Permissions From b0dcf579ef9a198a5c562c1171212a924eefc2c5 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:57:30 +0000 Subject: [PATCH 05/12] mdcode: recover relationships and deployment targets on pull Pull previously dropped two things push had already written to Knowledge Catalog: the model's deployment targets (stored in the semantic-model aspect) and its 1:1/1:N relationships (stored as schema-join entry links). The reader only opened per-entry aspects and pull only fetched entries, so both were silently lost even though the catalog held them. - kc_converter: read deploymentTargets back into the GOOGLE custom_extensions block, and invert schema-join links into Relationships -- endpoints resolved by data source, FK direction and columns from the join aspect. modelsFromCatalogResources grows an entryLinks argument. Relationship names come back normalized (lowercased/hyphenated): the emitter stores the name only in the link id. - pull_kc: add a second fetch pass over the entity entries via lookupEntryLinks, deduping the undirected links (each is returned from both endpoints). - Tests cover endpoint/direction recovery, name normalization, the M:N drop, deployment-target recovery, and the pull fetch+dedup path; the .pull.golden fixtures are regenerated and the docs pull note rewritten. M:N (association) relationships remain unrecovered -- push never emits them. Writer files are untouched. --- toolbox/mdcode/docs/semantic-model.md | 28 +- .../mdcode/src/libts/semantic/kc_converter.ts | 210 +++++++++++--- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 76 +++++- .../sales_bq_graph_target.pull.golden.yaml | 3 + .../star_orders_customer.pull.golden.yaml | 11 + .../fixtures/tpcds_date_edge.pull.golden.yaml | 29 ++ .../tests/libts/semantic/kc_converter.test.ts | 257 +++++++++++++----- .../tests/libts/semantic/pull_kc.test.ts | 107 +++++++- 8 files changed, 583 insertions(+), 138 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 80b7db91..ec67acbd 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -259,18 +259,22 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull is lossy.** It reconstructs a model only from what push wrote -> to the catalog (see the note under [What gets created in Knowledge -> Catalog](#what-gets-created-in-knowledge-catalog)), and recovers even less -> than the catalog holds. A pulled document comes back without entity keys, -> `ai_context`, field labels, the original vendor SQL (`importedExpression`), -> relationships (even the 1:1 / 1:N `schema-join` links push wrote — the edges -> live in the BigQuery property graph), and the `deploymentTargets` custom -> extension. A field's *role* survives as a bare `dimension: {}` marker, but its -> detail (`is_time`, and so on) does not. **A push followed by a pull does not -> return your original file** — treat a pulled document as a faithful copy of -> the catalog metadata, not of the authored model, and keep the authored -> document as the source of truth. +> **Note — pull is lossy.** It reconstructs a model from what push wrote to the +> catalog (see the note under [What gets created in Knowledge +> Catalog](#what-gets-created-in-knowledge-catalog)), so it recovers the model +> structure — entities and fields, metrics, 1:1 / 1:N relationships (from the +> `schema-join` links), and the deployment targets. It does **not** recover the +> content the catalog never held: entity keys, `ai_context`, field labels, the +> original vendor SQL (`importedExpression`), and M:N relationships (whose edge +> lives only in the BigQuery property graph). Some recovered content also comes +> back normalized rather than verbatim: relationship names are +> lowercased/hyphenated (the catalog stores the name only in the link id), a +> field's *role* survives as a bare `dimension: {}` marker without its detail +> (`is_time`, and so on), and a metric authored without a datatype comes back as +> an explicit `Decimal`. **A push followed by a pull does not return your original +> file** — treat a pulled document as a faithful copy of the catalog metadata, +> not of the authored model, and keep the authored document as the source of +> truth. ## Permissions diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 25a67c0b..89dd0ad5 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -1,39 +1,47 @@ // Knowledge Catalog <-> Semantic Model IR converter. // // SCAFFOLD (naming for the end state): this file currently holds only the -// READ direction -- Knowledge Catalog entries -> IR (`modelsFromCatalogResources` -// and its helpers). The WRITE direction (IR -> KC entries, -// `generateCatalogResources`) still lives in `knowledge_catalog.ts` and moves -// here once PR4 (the KC push line) merges, at which point this becomes the full -// two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is -// the KC emit code?" -> `knowledge_catalog.ts`. +// READ direction -- Knowledge Catalog entries -> IR +// (`modelsFromCatalogResources` and its helpers). The WRITE direction (IR -> KC +// entries, `generateCatalogResources`) still lives in `knowledge_catalog.ts` +// and moves here once PR4 (the KC push line) merges, at which point this +// becomes the full two-way converter and `knowledge_catalog.ts` goes away. +// Until then, "where is the KC emit code?" -> `knowledge_catalog.ts`. // // TODO(#278): fold the KC WRITE direction in and drop the scaffold. Once // #278 merges, move `generateCatalogResources` (and the `KcResources` type // + emit helpers) out of `knowledge_catalog.ts` into this file, delete -// `knowledge_catalog.ts`, and repoint its importers (`deploy_knowledge_catalog.ts` -// and the emitter tests). `idOf` -- shared by both directions and re-exported -// from here only for `pull_kc` today -- becomes a plain local. Then this is -// the sole KC<->IR codec and the SCAFFOLD note above comes out. +// `knowledge_catalog.ts`, and repoint its importers +// (`deploy_knowledge_catalog.ts` and the emitter tests). `idOf` -- shared by +// both directions and re-exported from here only for `pull_kc` today -- becomes +// a plain local. Then this is the sole KC<->IR codec and the SCAFFOLD note +// above comes out. // // The reader is the inverse of `generateCatalogResources`: it reconstructs the // IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). // `semantic-entity` / `semantic-metric` entries are grouped under their -// `semantic-model` anchor via `parentEntry`; entries of other types are ignored. -// Resources are matched by type-name SUFFIX, so a reader need not know which -// system-type project/location the emitter used. +// `semantic-model` anchor via `parentEntry`; entries of other types are +// ignored. Resources are matched by type-name SUFFIX, so a reader need not know +// which system-type project/location the emitter used. // -// Fidelity is bounded by what the emitter persisted, so this read is the inverse -// of the WRITE, not of the authored document. It recovers names, descriptions, -// data sources, field datatypes (via the schema aspect) and DIMENSION roles, -// field/metric expressions, and each metric's attach entity (re-derived from its -// expression, as the loader does). It cannot recover what the emitter does not -// write: entity keys/unique keys, `ai_context`, field labels, `importedDialect`, -// `custom_extensions`, and relationships (the graph edges live in the BigQuery -// property graph, not the catalog). - -import type {Entry} from '../gcp/dataplex'; -import {DataType, Entity, Field, Metric, SemanticModel} from './ir'; +// Fidelity is bounded by what the emitter persisted, so this read is the +// inverse of the WRITE, not of the authored document. It recovers names, +// descriptions, data sources, field datatypes (via the schema aspect) and +// DIMENSION roles, field/metric expressions, each metric's attach entity +// (re-derived from its expression, as the loader does), the model's deployment +// targets (from the semantic-model aspect, back into the GOOGLE +// `custom_extensions` block), and 1:1 / 1:N relationships (from the +// `schema-join` entry links a pull fetched -- see +// `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what +// the emitter does not write: entity keys/unique keys, `ai_context`, field +// labels, `importedDialect`, and many-to-many (association) relationships +// (whose edge lives only in the BigQuery property graph). Relationship NAMES +// come back normalized (lowercased/hyphenated), since the emitter encodes the +// name only in the link id (via `linkSlug`), not in the join aspect. + +import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; + +import {CustomExtension, DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; import {referencedEntityNames} from './sql_expr_utils'; export interface ReadResult { @@ -48,8 +56,14 @@ export interface ReadResult { * an orphaned child, an entry missing its aspect data). Entries must already be * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a * BASIC list omits aspect data, so the puller re-fetches each entry first. + * + * `entryLinks` are the `schema-join` links a pull fetched for the group (see + * `pull_kc`); each link between two of a model's entity entries is + * reconstructed as a 1:1 / 1:N relationship. Pass `[]` (the default) to + * reconstruct entities and metrics only. */ -export function modelsFromCatalogResources(entries: Entry[]): ReadResult { +export function modelsFromCatalogResources( + entries: Entry[], entryLinks: EntryLink[] = []): ReadResult { const warnings: string[] = []; const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); @@ -75,17 +89,26 @@ export function modelsFromCatalogResources(entries: Entry[]): ReadResult { const models = anchors.map(anchor => { const name = anchor.entrySource?.displayName ?? idOf(anchor.name); - const entities = childrenOf(anchor.name, entityEntries) - .map(e => readEntity(e, warnings)); + const entityEntriesForModel = childrenOf(anchor.name, entityEntries); + const entities = entityEntriesForModel.map(e => readEntity(e, warnings)); const entityNames = entities.map(e => e.name); const metrics = childrenOf(anchor.name, metricEntries) .map(e => readMetric(e, entityNames, warnings)); - // Relationships are not published to the catalog (see the file header), so - // a reconstructed model always has an empty edge set. - const model: SemanticModel = {name, entities, relationships: [], metrics}; + // Relationships come from the schema-join entry links whose two endpoints + // are both this model's entity entries (M:N edges were never published -- + // they live only in the BigQuery property graph -- so stay absent here). + const relationships = readRelationships( + entryLinks, name, new Set(entityEntriesForModel.map(e => e.name)), + dataSourceIndex(entities), warnings); + + const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; if (description !== undefined) model.description = description; + // Deployment targets ride back in the same GOOGLE custom_extensions block + // the author wrote them in (the inverse of the emitter's modelAspectData). + const targets = readDeploymentTargets(anchor); + if (targets) model.customExtensions = [targets]; return model; }); @@ -248,13 +271,21 @@ function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| } -// The `data` payload of an entry's aspect of the given bare type, matched by -// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` -// suffix (robust to whichever system-type project/location the emitter used). -// Returns an empty object when the aspect is absent. +// The `data` payload of an entry's aspect of the given bare type. See +// aspectDataOf; entries and entry links carry aspects in the same shape. function aspectData(entry: Entry, type: string): Record { - const aspects = entry.aspects ?? {}; - for (const [key, aspect] of Object.entries(aspects)) { + return aspectDataOf(entry.aspects, type); +} + + +// The `data` payload of an aspect of the given bare type from an aspect map, +// matched by the aspect key's `.` suffix or the aspectType's +// `/aspectTypes/` suffix (robust to whichever system-type +// project/location the emitter used). Returns an empty object when the aspect +// is absent. +function aspectDataOf(aspects: Record|undefined, type: string): + Record { + for (const [key, aspect] of Object.entries(aspects ?? {})) { if (key.endsWith(`.${type}`) || aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { return aspect.data ?? {}; @@ -264,6 +295,113 @@ function aspectData(entry: Entry, type: string): Record { } +// Recovers the model's deployment targets from its semantic-model aspect back +// into the GOOGLE custom_extension the author declared them in (the inverse of +// the emitter's modelAspectData). Returns undefined when the model has none. +function readDeploymentTargets(anchor: Entry): CustomExtension|undefined { + const targets = + asArray(aspectData(anchor, 'semantic-model').deploymentTargets) + .filter((t): t is string => typeof t === 'string' && t !== ''); + if (!targets.length) return undefined; + return { + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }; +} + + +// Indexes reconstructed entities by their SQL data source, so a schema-join +// aspect (which names each side by its table, not the entity) can resolve its +// endpoints back to entity names. +function dataSourceIndex(entities: Entity[]): Map { + const index = new Map(); + for (const entity of entities) { + const dataSource = (entity.dataSource ?? '').trim(); + if (dataSource) index.set(dataSource, entity.name); + } + return index; +} + + +// Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link +// whose two endpoints are both this model's entity entries becomes one +// direct-FK relationship. The aspect encodes direction (`source` is the +// foreign-key side) and the paired join columns; the link id encodes the +// (normalized) name. Links are deduped by name -- schema-join is undirected, so +// a per-entry lookup can return the same link once from each endpoint. +function readRelationships( + links: EntryLink[], modelName: string, entityEntryNames: Set, + dataSourceToEntity: Map, + warnings: string[]): Relationship[] { + const prefix = linkNamePrefix(modelName); + const seen = new Set(); + const out: Relationship[] = []; + for (const link of links) { + if (!link.entryLinkType?.endsWith('/entryLinkTypes/schema-join')) continue; + const refs = link.entryReferences ?? []; + // Only a link whose BOTH endpoints are this model's entities belongs here. + if (refs.length !== 2 || !refs.every(r => entityEntryNames.has(r.name))) { + continue; + } + if (link.name) { + if (seen.has(link.name)) continue; + seen.add(link.name); + } + const join = asArray(aspectDataOf(link.aspects, 'schema-join').joins)[0]; + if (!join) { + warnings.push( + `entry link '${link.name}': no schema-join aspect data; the ` + + `relationship is skipped`); + continue; + } + const source = dataSourceToEntity.get((join.source?.name ?? '').trim()); + const destination = + dataSourceToEntity.get((join.target?.name ?? '').trim()); + if (!source || !destination) { + warnings.push( + `entry link '${link.name}': a join endpoint table does not match a ` + + `reconstructed entity; the relationship is skipped`); + continue; + } + const rel: Relationship = { + name: relationshipName(link.name, prefix), + source: {entity: source, columns: asArray(join.source?.fields)}, + destination: {entity: destination, columns: asArray(join.target?.fields)}, + }; + if (join.description !== undefined) rel.description = join.description; + out.push(rel); + } + return out; +} + + +// Best-effort recovery of the relationship name from the link id, which the +// emitter built as linkSlug(`-`) (lowercased/hyphenated -- see +// knowledge_catalog.ts). Strips the model prefix when present; the name comes +// back normalized, never verbatim. +function relationshipName( + linkName: string|undefined, modelPrefix: string): string { + const id = idOf(linkName ?? ''); + if (modelPrefix && id.startsWith(`${modelPrefix}-`)) { + const rest = id.slice(modelPrefix.length + 1); + if (rest) return rest; + } + return id; +} + + +// Mirrors the model-name portion of the emitter's linkSlug so the prefix a link +// id was built with can be stripped. Kept local rather than imported: this +// read-side module must not depend on the write-side emitter. +function linkNamePrefix(modelName: string): string { + return modelName.toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^[^a-z]+/, '') + .replace(/^-+|-+$/g, ''); +} + + // The id segment of a full entry resource name (after the last '/'). export function idOf(name: string): string { return name.split('/').pop() ?? name; diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index aebaa7b5..24d338c3 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -16,7 +16,8 @@ // the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no // logic moves. -import {CatalogClient, Entry} from '../gcp/dataplex'; +import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; + import {SemanticModel} from './ir'; import {idOf, modelsFromCatalogResources} from './kc_converter'; @@ -65,9 +66,8 @@ export async function pullKnowledgeCatalog( if (!scoped.length) { return { models: [], - warnings: [ - `no semantic model named '${opts.model}' found in ${destination}` - ], + warnings: + [`no semantic model named '${opts.model}' found in ${destination}`], }; } } @@ -93,7 +93,45 @@ export async function pullKnowledgeCatalog( warnings.push(r.warning); } - const read = modelsFromCatalogResources(hydrated); + // Second fetch pass: relationships are schema-join entry links, which the + // entry list/lookup does not return. The catalog exposes links only per + // referenced entry (:lookupEntryLinks), so fan out over the entity entries + // and dedup by link name -- schema-join is undirected, so each link comes + // back once from each of its two endpoints. + const entityEntries = hydrated.filter( + e => e.entryType?.endsWith('/entryTypes/semantic-entity')); + const linkResults = + await mapConcurrent(entityEntries, HYDRATE_CONCURRENCY, async entry => { + const linkType = schemaJoinLinkType(entry.entryType); + const res = await cat.lookupEntryLinks(opts.project, opts.location, { + entry: entry.name, + entryLinkTypes: linkType ? [linkType] : undefined, + }); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry links for '${entry.name}' (status ${ + res.status}); relationships may be incomplete`, + }; + } + return {links: res.result}; + }); + + const seenLinks = new Set(); + const entryLinks: EntryLink[] = []; + for (const r of linkResults) { + if (r.warning) { + warnings.push(r.warning); + continue; + } + for (const link of r.links ?? []) { + const key = link.name ?? ''; + if (key && seenLinks.has(key)) continue; + if (key) seenLinks.add(key); + entryLinks.push(link); + } + } + + const read = modelsFromCatalogResources(hydrated, entryLinks); warnings.push(...read.warnings); // Defense in depth: keep only the requested model even if the reader surfaced @@ -136,6 +174,19 @@ function semanticAspectTypes(entryType: string): string[]|undefined { } +// The schema-join entry link type resource, derived from an entity's entryType +// base (the link type is the parallel resource in the same project/location the +// emitter referenced). Used to filter :lookupEntryLinks to just the +// relationship links. Returns undefined for an entryType with no recognizable +// base. +function schemaJoinLinkType(entryType: string): string|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + return `${entryType.slice(0, idx)}/entryLinkTypes/schema-join`; +} + + // Restricts hydration targets to a single model: the semantic-model anchor // whose name (entrySource.displayName, else the entry id) matches `model`, plus // every child entry whose parentEntry is that anchor. Uses only list-level @@ -146,13 +197,14 @@ function scopeToModel( model: string): {entry: Entry; aspectTypes: string[]}[] { const isAnchor = (t: {entry: Entry}) => !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); - const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); - const matchedAnchorNames = new Set( - targets.filter(isAnchor) - .filter( - t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === - model) - .map(t => t.entry.name)); + const allAnchorNames = + new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = + new Set(targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? + idOf(t.entry.name)) === model) + .map(t => t.entry.name)); if (!matchedAnchorNames.size) return []; // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml index f9e42580..06dd8bdc 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -2,6 +2,9 @@ version: 0.2.0.dev0 semantic_model: - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' datasets: - name: orders source: demo.sales.orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml index bf7cfc8d..d7a9cbe1 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -3,6 +3,9 @@ version: 0.2.0.dev0 semantic_model: - name: sales description: Sales orders with customer attributes + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' datasets: - name: orders source: samples.tpch.orders @@ -44,6 +47,14 @@ semantic_model: - dialect: BIGQUERY expression: c_name description: Customer name + relationships: + - name: orders-to-customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey metrics: - name: total_revenue expression: diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml index 7437f8ac..1900fe12 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -145,3 +145,32 @@ semantic_model: - name: date_dim source: sqlgen-testing.demo.date_dim description: Date dimension with calendar attributes + relationships: + - name: store-sales-to-date-dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store-sales-to-customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store-sales-to-item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store-sales-to-store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index d5ecb145..0e2f0eb6 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -2,22 +2,26 @@ // (modelsFromCatalogResources in src/libts/semantic/kc_converter.ts). // // The reader is the inverse of the emitter (generateCatalogResources). The -// central guarantee is an emitter -> reader round trip: emit a model's entries, -// read them back, and get an IR equal to the source WHERE the emitter is -// lossless. The write drops content by design (entity keys, ai_context, field -// labels, importedDialect, relationships -- see the emitter header), so the -// expected read-back is the source model with exactly those fields cleared. +// central guarantee is an emitter -> reader round trip: emit a model's entries +// AND entry links, read them back, and get an IR equal to the source WHERE the +// emitter is lossless. The write drops content by design (entity keys, +// ai_context, field labels, importedDialect, and many-to-many relationships -- +// see the emitter header), so the expected read-back is the source model with +// exactly those fields cleared. 1:1 / 1:N relationships and deployment targets +// DO round-trip (via schema-join links and the semantic-model aspect), except +// that relationship names come back normalized (lowercased/hyphenated). // Targeted tests pin the mapping details a round trip cannot isolate (the // dataType inverse, the DIMENSION role, resource-URI parsing, metric attach -// re-derivation, and parent/anchor grouping). +// re-derivation, relationship endpoint/direction recovery, and parent/anchor +// grouping). import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; import {serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -28,11 +32,11 @@ const OPTS = { entryGroup: 'eg' }; -// Emits a model to entries and reads it straight back. +// Emits a model to entries + entry links and reads it straight back. function roundTrip(model: SemanticModel): {models: SemanticModel[]; warnings: string[]} { - const {entries} = generateCatalogResources(model, OPTS); - return modelsFromCatalogResources(entries); + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources(entries, entryLinks); } @@ -75,6 +79,120 @@ describe('emitter -> reader round trip (lossless slice)', () => { }); +describe('relationship recovery (schema-join links -> IR)', () => { + // orders.o_custkey (the foreign-key side) references customer.c_custkey. + const twoEntities: Entity[] = [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ]; + + test( + 'a 1:N relationship recovers its endpoints, direction, and columns', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: + 'places', // already link-slug-safe, so it round-trips exactly + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const rels = roundTrip(model).models[0].relationships; + expect(rels).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'a relationship name comes back normalized (lowercased/hyphenated)', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'Places_Order', // mixed case + underscore -> normalized on + // read + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('places-order'); + }); + + test('a many-to-many (association) relationship is not recovered', () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'enrolls', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + association: { + dataSource: 'p.d.junction', + keys: ['id'], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + }, + }], + metrics: [], + }; + // The emitter never publishes M:N, so no schema-join link exists to read. + expect(roundTrip(model).models[0].relationships).toEqual([]); + }); +}); + + +describe( + 'deployment-target recovery (semantic-model aspect -> custom_extensions)', + () => { + test('the GOOGLE deployment targets ride back verbatim', () => { + const uri = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g'; + const data = JSON.stringify({deploymentTargets: [uri]}); + const model: SemanticModel = { + name: 'sales', + customExtensions: [{vendorName: 'GOOGLE', data}], + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toEqual([ + {vendorName: 'GOOGLE', data} + ]); + }); + + test( + 'a model with no deployment targets recovers no custom_extensions', + () => { + const model: SemanticModel = { + name: 'sales', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toBeUndefined(); + }); + }); + + describe('dataType inverse (schema aspect -> IR type)', () => { // Emit a one-field model of each IR type, read it back, and check the field's // reconstructed type. String and Opaque both emit dataType STRING; String @@ -246,21 +364,22 @@ describe('anchor / parent grouping', () => { describe('metric expression referencing no known entity', () => { - test('warns that the metric may be unplaceable and leaves it unattached', - () => { - const model: SemanticModel = { - name: 'm', - entities: - [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], - relationships: [], - // References `widgets`, which is not an entity of this model. - metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], - }; - const {models, warnings} = roundTrip(model); - expect(models[0].metrics[0].entity).toBeUndefined(); - expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) - .toBe(true); - }); + test( + 'warns that the metric may be unplaceable and leaves it unattached', + () => { + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], + relationships: [], + // References `widgets`, which is not an entity of this model. + metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].metrics[0].entity).toBeUndefined(); + expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) + .toBe(true); + }); }); @@ -268,48 +387,52 @@ describe('metric expression referencing no known entity', () => { // // The round trip above proves the reader inverts the emitter in memory; this // pins the reviewable artifact. For each corpus fixture it reads the committed -// emitter golden (`.knowledge_catalog.golden.json` -- the exact entries -// a push produced) back through the reader and serializes the reconstructed IR -// to `.pull.golden.yaml`. Open that next to the fixture's -// `.osi.golden.yaml` to see, as whole files, what a Knowledge Catalog round trip -// preserves and what it drops (keys, ai_context, labels, relationships). +// emitter golden (`.knowledge_catalog.golden.json` -- the exact +// entries and entry links a push produced) back through the reader and +// serializes the reconstructed IR to `.pull.golden.yaml`. Open that +// next to the fixture's `.osi.golden.yaml` to see, as whole files, what a +// Knowledge Catalog round trip preserves (including 1:1 / 1:N relationships and +// deployment targets) and what it drops (keys, ai_context, labels, vendor SQL, +// M:N relationships). // // Regenerate after an intentional reader/serializer change: // UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts -describe('golden pull: each corpus KC golden reconstructs to its exact YAML', - () => { - const CORPUS = [ - 'sales_bq_graph_target.yaml', - 'star_orders_customer.yaml', - 'tpcds_date_edge.yaml', - ]; - const kcGoldenPath = (fixture: string) => path.join( - FIXTURES, - fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); - const pullGoldenPath = (fixture: string) => - path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); - - for (const fixture of CORPUS) { - test(fixture, () => { - const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); - const {models, warnings} = modelsFromCatalogResources(kc.entries); - // Reader warnings ride along as YAML comments so the golden shows - // the full outcome, not just the recovered document. - const header = warnings.length ? - warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : - '# (no warnings)\n'; - const actual = - header + models.map(m => serializeModel(m).yaml).join('---\n'); - const golden = pullGoldenPath(fixture); - if (process.env.UPDATE_GOLDENS) { - fs.writeFileSync(golden, actual); - return; - } - if (!fs.existsSync(golden)) { - throw new Error(`missing golden ${ - path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); - } - expect(actual).toBe(fs.readFileSync(golden, 'utf8')); - }); - } - }); +describe( + 'golden pull: each corpus KC golden reconstructs to its exact YAML', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + const kcGoldenPath = (fixture: string) => path.join( + FIXTURES, + fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + const pullGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const {models, warnings} = + modelsFromCatalogResources(kc.entries, kc.entryLinks ?? []); + // Reader warnings ride along as YAML comments so the golden shows + // the full outcome, not just the recovered document. + const header = warnings.length ? + warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : + '# (no warnings)\n'; + const actual = + header + models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = pullGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename( + golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index 5a09cebd..d5d22d42 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -6,18 +6,19 @@ // the IR. The catalog client is stubbed so no network call is made. The entries // the fake serves are produced by the real emitter, so this exercises the true // list -> hydrate -> read path end to end (an entity is re-fetched with BOTH -// its semantic-entity and schema aspects). The reader's own mapping is covered -// in kc_converter.test.ts; the focus here is the fetch SEQUENCE: -// aspect hydration, the --model filter, skipped entries, and ignoring foreign -// entries. +// its semantic-entity and schema aspects, and a second pass fetches each +// entity's schema-join links). The reader's own mapping is covered in +// kc_converter.test.ts; the focus here is the fetch SEQUENCE: aspect hydration, +// the relationship-link fetch (per-entry, deduped across endpoints), the +// --model filter, skipped entries, and ignoring foreign entries. import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; import {ApiResult} from '../../../src/libts/gcp/api'; -import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; -import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; +import {CatalogClient, Entry, EntryLink} from '../../../src/libts/gcp/dataplex'; import {SemanticModel} from '../../../src/libts/semantic/ir'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; const OPTS = { project: 'dest', @@ -60,8 +61,13 @@ function err(status: number, message: string): ApiResult { // Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, // or 404s an unknown name). `lookupFail` forces a failure for one entry name. -function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): - {list: any; lookup: any} { +// `links` are served by lookupEntryLinks per referenced entry (so an undirected +// link comes back once from each endpoint, exercising the puller's dedup); +// `linkFail` forces a link-lookup failure for one entry name. +function stubClient( + listed: Entry[], served: Entry[], lookupFail?: string, + links: EntryLink[] = [], + linkFail?: string): {list: any; lookup: any; lookupLinks: any} { const byName = new Map(served.map(e => [e.name, e])); const list = spyOn(CatalogClient.prototype, 'listEntries') .mockImplementation(async function*() { @@ -73,7 +79,16 @@ function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): const e = byName.get(name); return e ? ok(e) : err(404, 'not found'); }); - return {list, lookup}; + const lookupLinks = + spyOn(CatalogClient.prototype, 'lookupEntryLinks') + .mockImplementation(async (_p: any, _l: any, opts: any) => { + if (opts.entry === linkFail) return err(500, 'boom'); + const forEntry = links.filter( + l => + (l.entryReferences ?? []).some(r => r.name === opts.entry)); + return ok(forEntry); + }); + return {list, lookup, lookupLinks}; } afterEach(() => { @@ -121,6 +136,76 @@ describe('pullKnowledgeCatalog: happy path', () => { }); +describe('pullKnowledgeCatalog: relationship links', () => { + // A 1:N model: orders.o_custkey (the FK side) references customer.c_custkey. + const SALES_REL: SemanticModel = { + name: 'sales', + entities: [ + { + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'demo.sales.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ], + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + + test( + 'a schema-join link reconstructs into a relationship exactly once', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const {lookupLinks} = + stubClient(entries, entries, undefined, entryLinks); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + expect(warnings).toHaveLength(0); + // Links are fetched per entity (2 entities), and the single undirected + // link -- returned from both endpoints -- is deduped to one edge. + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + + test( + 'a failed link lookup on one endpoint still recovers the edge from the ' + + 'other, and warns', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const ordersEntry = + entries.find(e => (e.entrySource?.displayName) === 'orders')!; + // Fail the link lookup for the orders entity; the link still comes back + // from the customer endpoint. + stubClient(entries, entries, undefined, entryLinks, ordersEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships.map(r => r.name)).toEqual(['places']); + expect(warnings.some( + w => /failed to fetch entry links/i.test(w) && + w.includes(ordersEntry.name))) + .toBe(true); + }); +}); + + describe('pullKnowledgeCatalog: filtering and robustness', () => { test('--model keeps only the named model', async () => { const other: SemanticModel = { @@ -215,8 +300,8 @@ describe('pullKnowledgeCatalog: filtering and robustness', () => { // drops a child a full pull returns). const metricEntry = entries.find(e => e.entryType.endsWith('/semantic-metric'))!; - metricEntry.parentEntry = - metricEntry.parentEntry!.replace('projects/dest/', 'projects/12345/'); + metricEntry.parentEntry = metricEntry.parentEntry!.replace( + 'projects/dest/', 'projects/12345/'); const {lookup} = stubClient(entries, entries); const cat = new CatalogClient({} as any); From e1982e6d79f224946320d4163bce05be2d481a18 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:10:56 +0000 Subject: [PATCH 06/12] mdcode: resolve pull relationship endpoints via entryReferences Address code-review findings on the gap-3 pull leg: - Resolve schema-join endpoints from the link's entryReferences, matched by entry id, instead of a dataSource->entity index. The id is unique per entity (fixes two entities sharing a table collapsing last-wins) and is stable across the project-number/id normalization lookupEntry applies to entries but lookupEntryLinks does not apply to link references (fixes relationships silently dropping on the live path). The schema-join aspect is now used only for FK direction + join columns; undecidable direction keeps the reference order and warns rather than dropping the edge. - Dedup entry links by a sorted endpoint-pair key when a link has no name (shared linkDedupKey, reused by pull_kc) so a nameless link returned from both endpoints is not counted twice. - Rewrite the pull 'lossy' note in the user guide as recovered-exactly / recovered-but-normalized / not-recovered bullets. Tests: shared-table endpoints, un-normalized project-number references, prefix-stripping across tricky model names, and nameless-link dedup. --- toolbox/mdcode/docs/semantic-model.md | 47 ++++--- .../mdcode/src/libts/semantic/kc_converter.ts | 111 ++++++++++------ toolbox/mdcode/src/libts/semantic/pull_kc.ts | 10 +- .../tests/libts/semantic/kc_converter.test.ts | 123 ++++++++++++++++++ .../tests/libts/semantic/pull_kc.test.ts | 19 +++ 5 files changed, 250 insertions(+), 60 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index ec67acbd..b3a1fcbb 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -259,22 +259,37 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull is lossy.** It reconstructs a model from what push wrote to the -> catalog (see the note under [What gets created in Knowledge -> Catalog](#what-gets-created-in-knowledge-catalog)), so it recovers the model -> structure — entities and fields, metrics, 1:1 / 1:N relationships (from the -> `schema-join` links), and the deployment targets. It does **not** recover the -> content the catalog never held: entity keys, `ai_context`, field labels, the -> original vendor SQL (`importedExpression`), and M:N relationships (whose edge -> lives only in the BigQuery property graph). Some recovered content also comes -> back normalized rather than verbatim: relationship names are -> lowercased/hyphenated (the catalog stores the name only in the link id), a -> field's *role* survives as a bare `dimension: {}` marker without its detail -> (`is_time`, and so on), and a metric authored without a datatype comes back as -> an explicit `Decimal`. **A push followed by a pull does not return your original -> file** — treat a pulled document as a faithful copy of the catalog metadata, -> not of the authored model, and keep the authored document as the source of -> truth. +> **Note — pull reconstructs what the catalog holds, not your original file.** +> Pull can only recover what push wrote (see the note under [What gets created in +> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)). What that means in +> practice: +> +> **Recovered exactly** — these come back as authored: +> - Model structure: the model, its entities, and each entity's fields. +> - Field data source and data type; the field expression. +> - Metrics: name, expression, data type, and attach entity. +> - 1:1 / 1:N relationships: endpoints, foreign-key direction, and join columns +> (from the `schema-join` links). +> - Deployment targets. +> +> **Recovered, but normalized** — the content survives, the form changes: +> - Relationship *names* come back lowercased/hyphenated (the catalog stores the +> name only in the link id, e.g. `Places Order` → `places-order`). +> - A field marked as a dimension comes back as a bare `dimension: {}` marker, +> without its detail (`is_time`, and so on). +> - A metric authored with no data type comes back as an explicit `Decimal` +> (push must write a type, and defaults it to `NUMERIC`). +> +> **Not recovered** — push never wrote these, so pull cannot return them: +> - Entity keys / unique keys. +> - `ai_context`. +> - Field labels. +> - The original vendor SQL (`importedExpression`). +> - M:N relationships (the edge lives only in the BigQuery property graph). +> +> **So: a push followed by a pull does not return your original file.** Treat a +> pulled document as a faithful copy of the catalog metadata, not of the authored +> model, and keep the authored document as the source of truth. ## Permissions diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 89dd0ad5..3600d781 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -95,12 +95,21 @@ export function modelsFromCatalogResources( const metrics = childrenOf(anchor.name, metricEntries) .map(e => readMetric(e, entityNames, warnings)); + // Index this model's entities by their entry id, so a schema-join link can + // resolve its endpoints from the link's entryReferences. The id segment is + // stable across the project-number/id normalization that lookupEntry + // applies to entries but lookupEntryLinks does not apply to link + // references, so matching on it (not the full resource name) keeps + // live-fetched links resolvable. + const entityByEntryId = new Map(); + entityEntriesForModel.forEach( + (e, i) => entityByEntryId.set(idOf(e.name), entities[i])); + // Relationships come from the schema-join entry links whose two endpoints // are both this model's entity entries (M:N edges were never published -- // they live only in the BigQuery property graph -- so stay absent here). - const relationships = readRelationships( - entryLinks, name, new Set(entityEntriesForModel.map(e => e.name)), - dataSourceIndex(entities), warnings); + const relationships = + readRelationships(entryLinks, name, entityByEntryId, warnings); const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; @@ -310,28 +319,22 @@ function readDeploymentTargets(anchor: Entry): CustomExtension|undefined { } -// Indexes reconstructed entities by their SQL data source, so a schema-join -// aspect (which names each side by its table, not the entity) can resolve its -// endpoints back to entity names. -function dataSourceIndex(entities: Entity[]): Map { - const index = new Map(); - for (const entity of entities) { - const dataSource = (entity.dataSource ?? '').trim(); - if (dataSource) index.set(dataSource, entity.name); - } - return index; -} - - // Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link // whose two endpoints are both this model's entity entries becomes one -// direct-FK relationship. The aspect encodes direction (`source` is the -// foreign-key side) and the paired join columns; the link id encodes the -// (normalized) name. Links are deduped by name -- schema-join is undirected, so -// a per-entry lookup can return the same link once from each endpoint. +// direct-FK relationship. +// +// Endpoint identity comes from the link's entryReferences (matched by entry id +// via `entityByEntryId`), NOT from the aspect's table names: the id is unique +// per entity (two entities can share a table) and survives the +// project-number/id normalization lookupEntry applies to entries but +// lookupEntryLinks does not apply to link references. The aspect then supplies +// the join columns and the direction -- its `source` side is the foreign-key +// side, named by its data source -- used only to orient the two endpoints. The +// link id encodes the (normalized) relationship name. Links are deduped -- +// schema-join is undirected, so a per-entry lookup can return the same link +// once from each endpoint. function readRelationships( - links: EntryLink[], modelName: string, entityEntryNames: Set, - dataSourceToEntity: Map, + links: EntryLink[], modelName: string, entityByEntryId: Map, warnings: string[]): Relationship[] { const prefix = linkNamePrefix(modelName); const seen = new Set(); @@ -339,14 +342,18 @@ function readRelationships( for (const link of links) { if (!link.entryLinkType?.endsWith('/entryLinkTypes/schema-join')) continue; const refs = link.entryReferences ?? []; - // Only a link whose BOTH endpoints are this model's entities belongs here. - if (refs.length !== 2 || !refs.every(r => entityEntryNames.has(r.name))) { - continue; - } - if (link.name) { - if (seen.has(link.name)) continue; - seen.add(link.name); - } + if (refs.length !== 2) continue; + // Resolve both endpoints from the link's references. A reference that is + // not one of this model's entities (another model, or a non-entity) means + // the link does not belong here -- skip it quietly, as with M:N edges. + const endA = entityByEntryId.get(idOf(refs[0].name)); + const endB = entityByEntryId.get(idOf(refs[1].name)); + if (!endA || !endB) continue; + + const key = linkDedupKey(link); + if (seen.has(key)) continue; + seen.add(key); + const join = asArray(aspectDataOf(link.aspects, 'schema-join').joins)[0]; if (!join) { warnings.push( @@ -354,19 +361,35 @@ function readRelationships( `relationship is skipped`); continue; } - const source = dataSourceToEntity.get((join.source?.name ?? '').trim()); - const destination = - dataSourceToEntity.get((join.target?.name ?? '').trim()); - if (!source || !destination) { + // Direction lives in the aspect: `source` is the foreign-key side, named by + // its data source. Orient the two endpoints by matching that name, keeping + // join.source.fields paired with the source side. When neither orientation + // matches -- or both do, because the endpoints share a data source -- the + // direction is undecidable, so keep the reference order and warn rather + // than drop the edge. + const srcName = (join.source?.name ?? '').trim(); + const tgtName = (join.target?.name ?? '').trim(); + const aSrc = (endA.dataSource ?? '').trim(); + const bSrc = (endB.dataSource ?? '').trim(); + const endAIsSource = aSrc === srcName && bSrc === tgtName; + const endBIsSource = bSrc === srcName && aSrc === tgtName; + let [source, destination] = [endA, endB]; + if (endBIsSource && !endAIsSource) { + [source, destination] = [endB, endA]; + } else if (!endAIsSource && !endBIsSource) { warnings.push( - `entry link '${link.name}': a join endpoint table does not match a ` + - `reconstructed entity; the relationship is skipped`); - continue; + `entry link '${link.name}': join direction does not match either ` + + `endpoint's data source; using the reference order`); + } else if (endAIsSource && endBIsSource) { + warnings.push( + `entry link '${link.name}': join direction is ambiguous (endpoints ` + + `share a data source); using the reference order`); } const rel: Relationship = { name: relationshipName(link.name, prefix), - source: {entity: source, columns: asArray(join.source?.fields)}, - destination: {entity: destination, columns: asArray(join.target?.fields)}, + source: {entity: source.name, columns: asArray(join.source?.fields)}, + destination: + {entity: destination.name, columns: asArray(join.target?.fields)}, }; if (join.description !== undefined) rel.description = join.description; out.push(rel); @@ -375,6 +398,16 @@ function readRelationships( } +// A stable dedup key for an entry link: its name when present, else its +// (order-independent) endpoint pair and type. Guards against a per-endpoint +// lookup returning an unnamed link once from each of its two endpoints. +export function linkDedupKey(link: EntryLink): string { + if (link.name) return link.name; + const refs = (link.entryReferences ?? []).map(r => r.name).sort(); + return `${refs.join('')}${link.entryLinkType ?? ''}`; +} + + // Best-effort recovery of the relationship name from the link id, which the // emitter built as linkSlug(`-`) (lowercased/hyphenated -- see // knowledge_catalog.ts). Strips the model prefix when present; the name comes diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index 24d338c3..a1ebae76 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -19,7 +19,7 @@ import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; import {SemanticModel} from './ir'; -import {idOf, modelsFromCatalogResources} from './kc_converter'; +import {idOf, linkDedupKey, modelsFromCatalogResources} from './kc_converter'; export interface KcPullOptions { project: string; @@ -96,7 +96,7 @@ export async function pullKnowledgeCatalog( // Second fetch pass: relationships are schema-join entry links, which the // entry list/lookup does not return. The catalog exposes links only per // referenced entry (:lookupEntryLinks), so fan out over the entity entries - // and dedup by link name -- schema-join is undirected, so each link comes + // and dedup (linkDedupKey) -- schema-join is undirected, so each link comes // back once from each of its two endpoints. const entityEntries = hydrated.filter( e => e.entryType?.endsWith('/entryTypes/semantic-entity')); @@ -124,9 +124,9 @@ export async function pullKnowledgeCatalog( continue; } for (const link of r.links ?? []) { - const key = link.name ?? ''; - if (key && seenLinks.has(key)) continue; - if (key) seenLinks.add(key); + const key = linkDedupKey(link); + if (seenLinks.has(key)) continue; + seenLinks.add(key); entryLinks.push(link); } } diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 0e2f0eb6..88a58a22 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -156,6 +156,129 @@ describe('relationship recovery (schema-join links -> IR)', () => { // The emitter never publishes M:N, so no schema-join link exists to read. expect(roundTrip(model).models[0].relationships).toEqual([]); }); + + test( + 'endpoints resolve from the link references, not the shared table name', + () => { + // Two entities over the SAME table: a data-source index would collapse + // them (last write wins), so both endpoints must come from the link's + // entryReferences instead. Direction can't be told from the shared + // table, so the reader keeps the reference order and warns. + const sameTable: Entity[] = [ + { + name: 'parent_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'id', expression: 'parent_order.id'}], + }, + { + name: 'child_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'parent_id', expression: 'child_order.parent_id'}], + }, + ]; + const model: SemanticModel = { + name: 'sales', + entities: sameTable, + relationships: [{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }], + metrics: [], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].relationships).toEqual([{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }]); + expect(warnings.some(w => /join direction is ambiguous/i.test(w))) + .toBe(true); + }); + + test( + 'endpoints resolve when link references carry an un-normalized project ' + + 'number', + () => { + // The live path is asymmetric: lookupEntry normalizes entry names to + // project IDs, but lookupEntryLinks returns references still carrying + // the numeric project. Matching on the (stable) entry id keeps the + // relationship resolvable rather than silently dropping it. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const numeric = entryLinks.map( + l => ({ + ...l, + entryReferences: + (l.entryReferences ?? + []).map(r => ({ + ...r, + name: r.name.replace( + 'projects/dest/', 'projects/000000000000/'), + })), + })); + const {models} = modelsFromCatalogResources(entries, numeric); + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'the model prefix is stripped across tricky model names ' + + '(linkNamePrefix tracks the emitter slug)', + () => { + // Pins the read-side prefix reproduction to the emitter's linkSlug: if + // it drifts, the model prefix would survive and the recovered name + // would not equal the bare, normalized relationship name. + for (const modelName of ['Sales', 'Sales Analytics', '123 Sales!']) { + const model: SemanticModel = { + name: modelName, + entities: twoEntities, + relationships: [{ + name: 'rel_one', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('rel-one'); + } + }); + + test('an unnamed link returned from both endpoints is deduped', () => { + // Defense in depth: if the catalog ever returns a nameless schema-join + // link, the per-endpoint fan-out yields it twice; dedup falls back to the + // endpoint pair so it becomes one relationship, not two. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {models} = modelsFromCatalogResources( + entries, [...nameless, ...nameless.map(l => ({...l}))]); + expect(models[0].relationships).toHaveLength(1); + }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index d5d22d42..2316003d 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -183,6 +183,25 @@ describe('pullKnowledgeCatalog: relationship links', () => { expect(lookupLinks).toHaveBeenCalledTimes(2); }); + test( + 'an unnamed link returned from both endpoints is deduped to one edge', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + // A nameless link can't dedup by name; the puller must fall back to the + // endpoint pair, or the per-endpoint fan-out would yield it twice. + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {lookupLinks} = stubClient(entries, entries, undefined, nameless); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + + // Deduped to a single edge (a nameless link has no id, so the name + // itself can't be recovered -- but it must not be counted twice). + expect(models[0].relationships).toHaveLength(1); + expect(models[0].relationships[0].source.entity).toBe('orders'); + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + test( 'a failed link lookup on one endpoint still recovers the edge from the ' + 'other, and warns', From e6d5e1f97652082ebf944f2dd3fc8190bccb0e86 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:29:14 +0000 Subject: [PATCH 07/12] mdcode: assert pull symmetry + complete the sales BQ golden Two fixture-coverage gaps from the round-trip review: 1. Symmetry assertion. The golden pull files let a human eyeball what a Knowledge Catalog round trip drops, but nothing asserted it. Add a symmetry test over the converter corpus: load the authored IR, run a full emit -> read round trip, and assert the result equals the authored IR reduced to the "KC floor" (stripToKcFloor) -- the documented losses and normalizations applied to both sides. An undocumented regression (a dropped column, a lost description, an un-stripped M:N edge) now fails here even though each individual loss is already pinned by a targeted test. Export linkNamePrefix so the normalizer reproduces the relationship slug rather than reimplementing it. 2. sales_bq_graph_target had OSI/KC/pull goldens but no BigQuery golden. Add it to the BigQuery corpus and generate the golden: a valid single-node property graph with a MEASURE, so the fixture now carries a complete four-arm round-trip suite. No production behavior change; reader/emitter untouched apart from the linkNamePrefix export. --- .../mdcode/src/libts/semantic/kc_converter.ts | 6 +- .../tests/libts/semantic/bigquery.e2e.test.ts | 1 + .../sales_bq_graph_target.bigquery.golden.sql | 13 ++ .../tests/libts/semantic/kc_converter.test.ts | 134 +++++++++++++++++- 4 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 3600d781..7a04d7b2 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -425,8 +425,10 @@ function relationshipName( // Mirrors the model-name portion of the emitter's linkSlug so the prefix a link // id was built with can be stripped. Kept local rather than imported: this -// read-side module must not depend on the write-side emitter. -function linkNamePrefix(modelName: string): string { +// read-side module must not depend on the write-side emitter. Exported for the +// symmetry test, which reuses it to reproduce the normalized relationship name +// rather than reimplementing the slug rule. +export function linkNamePrefix(modelName: string): string { return modelName.toLowerCase() .replace(/[^a-z0-9-]+/g, '-') .replace(/-+/g, '-') diff --git a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts index b16be403..17816f02 100644 --- a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts @@ -41,6 +41,7 @@ const CORPUS = [ 'measure_lowering.yaml', 'metric_skips.yaml', 'keyless_dimension.yaml', + 'sales_bq_graph_target.yaml', ]; // Loads a fixture file to its IR. Split out from `build` so a test that only diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql new file mode 100644 index 00000000..51059fe9 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE PROPERTY GRAPH `sqlgen-testing.demo.sales` +NODE TABLES ( + `demo.sales.orders` AS orders + KEY(o_orderkey) + PROPERTIES( + o_orderkey, + o_totalprice, + MEASURE(SUM(o_totalprice)) AS total_revenue + ) +); + +-- warnings -- +-- note: no 'BIGQUERY' dialect for one or more expressions; using the portable 'ANSI_SQL' dialect verbatim ('BIGQUERY' accepts the ANSI core subset — supply 'BIGQUERY' variants only for BIGQUERY-specific SQL) diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 88a58a22..91967af8 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -19,9 +19,10 @@ import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {CustomExtension, Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {linkNamePrefix, modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {loadModels} from '../../../src/libts/semantic/loader'; import {serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -559,3 +560,132 @@ describe( }); } }); + + +// -- Symmetry: emit -> read drops ONLY the documented set. -- +// +// The golden pull above lets a human eyeball what a Knowledge Catalog round +// trip loses; this asserts it mechanically. For each corpus fixture it loads +// the authored IR, runs a full emit -> read round trip, and asserts the result +// equals the authored IR reduced to the "KC floor" -- the model with exactly +// the documented losses and normalizations applied (`stripToKcFloor`). Applying +// that same reduction to BOTH sides makes `toEqual` flag only UNDOCUMENTED +// divergence, so a new emitter/reader regression (a dropped column, a lost +// description, an un-stripped M:N edge) fails here even though every individual +// loss is already pinned by a targeted test above. +describe( + 'symmetry: an emit -> read round trip drops only documented fields', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the OSI / KC / pull goldens. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + for (const authored of loadModels(text, LOAD).models) { + const {models, warnings} = roundTrip(authored); + expect(models).toHaveLength(1); + expect(stripToKcFloor(models[0])).toEqual(stripToKcFloor(authored)); + // A clean corpus model round-trips without reader warnings. + expect(warnings).toEqual([]); + } + }); + } + }); + + +// Reduces a model to the "KC floor": the most an emit -> read round trip can +// preserve, i.e. the authored model with exactly the documented Knowledge +// Catalog losses and normalizations applied. Idempotent (already-floored input +// is unchanged), so it can be applied to both sides of a round trip. Each +// transform mirrors one emitter/reader behavior pinned by a targeted test +// above. +function stripToKcFloor(model: SemanticModel): SemanticModel { + const m = structuredClone(model); + + // Model ai_context is not persisted; only a GOOGLE deployment-targets block + // rides back, re-serialized to the reader's canonical JSON form. + delete m.aiContext; + const ext = canonicalGoogleExt(m.customExtensions); + if (ext) { + m.customExtensions = ext; + } else { + delete m.customExtensions; + } + + for (const e of m.entities) { + e.keys = []; // keys / unique keys are never persisted + delete e.uniqueKeys; + delete e.aiContext; + delete e.customExtensions; + for (const f of e.fields) { + delete f.label; // display label is not persisted + delete f.aiContext; + delete f.importedExpression; // vendor SQL is not persisted + delete f.importedDialect; + delete f.customExtensions; + if (f.dimension) f.dimension = {}; // only the DIMENSION role survives + // String is indistinguishable from an un-typed field on read. + if (f.type === 'String') delete f.type; + } + } + + for (const metric of m.metrics) { + delete metric.aiContext; + delete metric.importedExpression; + delete metric.importedDialect; + delete metric.customExtensions; + // The emitter writes a required dataType, defaulting typeless -> NUMERIC, + // which reads back as Decimal; String collapses to un-typed like fields. + if (metric.type === undefined) { + metric.type = 'Decimal'; + } else if (metric.type === 'String') { + delete metric.type; + } + } + + // M:N (association) edges are never published; a 1:1 / 1:N name comes back + // normalized via the emitter's slug (its exact form is pinned above). + m.relationships = m.relationships.filter(r => !r.association).map(r => { + const rel = structuredClone(r); + rel.name = linkNamePrefix(rel.name); + delete rel.aiContext; + delete rel.customExtensions; + return rel; + }); + + return m; +} + + +// The inverse-canonical of the emitter's deployment-target persistence, +// matching the reader's readDeploymentTargets: keep only GOOGLE +// deploymentTargets and re-serialize them so an authored block and its +// round-tripped form (which the reader always JSON.stringifies afresh) compare +// equal. Returns undefined when the model declares none, exactly as the reader +// omits custom_extensions then. +function canonicalGoogleExt(exts: CustomExtension[]|undefined): + CustomExtension[]|undefined { + const targets: string[] = []; + for (const ext of exts ?? []) { + if (ext.vendorName !== 'GOOGLE') continue; + let parsed: any; + try { + parsed = JSON.parse(ext.data); + } catch { + continue; + } + for (const t of parsed?.deploymentTargets ?? []) { + if (typeof t === 'string' && t) targets.push(t); + } + } + return targets.length ? [{ + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }] : + undefined; +} From efcc162eeb0c12aa0fb0fccc35785857954665c3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:40:48 +0000 Subject: [PATCH 08/12] mdcode: drop the sales_bq_graph_target BigQuery golden The BigQuery corpus golden path names the graph from the test's build opts (sqlgen-testing.demo.sales), ignoring the fixture's deployment target -- so the golden neither reflected the fixture's purpose nor matched a real deploy (demo.sales.sales_graph). That target-driven name is already covered by deploy_bigquery.test.ts, making this golden redundant. Revert the corpus addition and remove the generated file; the fixture keeps its OSI/KC/pull goldens and the pull symmetry assertion. --- .../tests/libts/semantic/bigquery.e2e.test.ts | 1 - .../sales_bq_graph_target.bigquery.golden.sql | 13 ------------- 2 files changed, 14 deletions(-) delete mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql diff --git a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts index 17816f02..b16be403 100644 --- a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts @@ -41,7 +41,6 @@ const CORPUS = [ 'measure_lowering.yaml', 'metric_skips.yaml', 'keyless_dimension.yaml', - 'sales_bq_graph_target.yaml', ]; // Loads a fixture file to its IR. Split out from `build` so a test that only diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql deleted file mode 100644 index 51059fe9..00000000 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE OR REPLACE PROPERTY GRAPH `sqlgen-testing.demo.sales` -NODE TABLES ( - `demo.sales.orders` AS orders - KEY(o_orderkey) - PROPERTIES( - o_orderkey, - o_totalprice, - MEASURE(SUM(o_totalprice)) AS total_revenue - ) -); - --- warnings -- --- note: no 'BIGQUERY' dialect for one or more expressions; using the portable 'ANSI_SQL' dialect verbatim ('BIGQUERY' accepts the ANSI core subset — supply 'BIGQUERY' variants only for BIGQUERY-specific SQL) From ff04848e30c91b875d6dfcf0c138639f36162ce7 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 15:41:56 +0000 Subject: [PATCH 09/12] mdcode: fix pull reader review findings Address code-review findings on the KC->IR reader (kc_converter.ts): - readMetric: when the expression does not pin exactly one known entity (none, or several), fall back to the attach entity metricAspectData persisted instead of dropping it. A cross-entity metric now recovers its authored entity. - readField: skip a schema field with no name (warn) instead of emitting a Field with an undefined name into the entity. - linkNamePrefix: mirror linkSlug's 63-char cap and trailing-hyphen re-strip so the read-side prefix stays aligned with the emitter's link id. - Header comment: add importedExpression (vendor SQL) and String/Opaque- typed metrics to the documented round-trip loss list. Add regression tests for the metric-entity fallback and the nameless-field skip. --- .../mdcode/src/libts/semantic/kc_converter.ts | 48 +++++++++++---- .../tests/libts/semantic/kc_converter.test.ts | 60 +++++++++++++++++++ 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 7a04d7b2..e732ad55 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -28,14 +28,18 @@ // inverse of the WRITE, not of the authored document. It recovers names, // descriptions, data sources, field datatypes (via the schema aspect) and // DIMENSION roles, field/metric expressions, each metric's attach entity -// (re-derived from its expression, as the loader does), the model's deployment -// targets (from the semantic-model aspect, back into the GOOGLE -// `custom_extensions` block), and 1:1 / 1:N relationships (from the -// `schema-join` entry links a pull fetched -- see +// (re-derived from its expression, as the loader does, else the value the +// emitter persisted), the model's deployment targets (from the semantic-model +// aspect, back into the GOOGLE `custom_extensions` block), and 1:1 / 1:N +// relationships (from the `schema-join` entry links a pull fetched -- see // `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what // the emitter does not write: entity keys/unique keys, `ai_context`, field -// labels, `importedDialect`, and many-to-many (association) relationships -// (whose edge lives only in the BigQuery property graph). Relationship NAMES +// labels, `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// many-to-many (association) relationships (whose edge lives only in the +// BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads +// back un-typed: the metric aspect persists only `dataType` (both collapse to +// `STRING`, with no metadataType to disambiguate), so -- as with a plain STRING +// field -- the reader leaves it un-typed rather than guess. Relationship NAMES // come back normalized (lowercased/hyphenated), since the emitter encodes the // name only in the link id (via `linkSlug`), not in the join aspect. @@ -158,7 +162,9 @@ function readEntity(entry: Entry, warnings: string[]): Entity { name, dataSource, keys: [], // not persisted by the emitter; unrecoverable on read - fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + fields: asArray(schema.fields) + .map(fd => readField(fd, name, warnings)) + .filter((f): f is Field => f !== undefined), }; const description = entry.entrySource?.description; if (description !== undefined) entity.description = description; @@ -170,12 +176,14 @@ function readEntity(entry: Entry, warnings: string[]): Entity { // schemaAspectData: the datatype from dataType/metadataType, expressions from // the nested `semantics` block, and the DIMENSION role back to a dimension // marker. -function readField(fd: any, entityName: string, warnings: string[]): Field { +function readField(fd: any, entityName: string, warnings: string[]): Field| + undefined { const name = fd?.name; if (name === undefined || name === '') { warnings.push( `entity '${entityName}': a schema field is missing its name; the ` + - `field may not load`); + `field is skipped`); + return undefined; } const field: Field = {name}; const sem = fd?.semantics ?? {}; @@ -203,8 +211,18 @@ function readMetric( if (data.expression !== undefined) metric.expression = data.expression; const exprForRefs = data.expression ?? ''; const referenced = referencedEntityNames(exprForRefs, entityNames); + const persistedEntity = + typeof data.entity === 'string' && data.entity !== '' ? data.entity : + undefined; if (referenced.length === 1) { + // The expression pins exactly one known entity; re-derive it (as the loader + // does) so the attach entity stays consistent with the reconstructed set. metric.entity = referenced[0]; + } else if (persistedEntity !== undefined) { + // The expression does not pin a single known entity (none, or several), so + // fall back to the attach entity the emitter persisted (metricAspectData) + // rather than dropping it. + metric.entity = persistedEntity; } else if (exprForRefs && !referenced.length) { // Parity with the loader's convertMetric: an expression that qualifies no // known entity is flagged as potentially unplaceable downstream. @@ -428,12 +446,22 @@ function relationshipName( // read-side module must not depend on the write-side emitter. Exported for the // symmetry test, which reuses it to reproduce the normalized relationship name // rather than reimplementing the slug rule. +// +// It reproduces linkSlug's normalization AND its 63-char cap + trailing-hyphen +// re-strip, so the prefix stays aligned with the id even when the combined +// `-` slug was truncated. (linkSlug's `|| 'link'` empty-slug +// fallback is intentionally omitted: an empty prefix strips nothing and +// relationshipName then returns the id verbatim, which is already correct. When +// the cap truncates into the model slug itself the relationship name is +// unrecoverable regardless -- relationshipName returns the truncated id.) export function linkNamePrefix(modelName: string): string { return modelName.toLowerCase() .replace(/[^a-z0-9-]+/g, '-') .replace(/-+/g, '-') .replace(/^[^a-z]+/, '') - .replace(/^-+|-+$/g, ''); + .replace(/^-+|-+$/g, '') + .slice(0, 63) + .replace(/-+$/, ''); } diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 91967af8..e63d6bf0 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -392,6 +392,35 @@ describe('field mapping details', () => { expect(back.importedExpression).toBeUndefined(); expect(back.importedDialect).toBeUndefined(); }); + + test('a schema field missing its name is skipped with a warning', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f'}], + }], + relationships: [], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + // Inject a nameless field record into the entity's schema aspect, as a + // malformed catalog could return; the reader must drop it, not emit a field + // with an undefined name. + for (const entry of entries) { + for (const aspect of Object.values(entry.aspects ?? {})) { + const fields = (aspect as {data?: {fields?: unknown[]}}).data?.fields; + if (Array.isArray(fields)) fields.push({dataType: 'INT64'}); + } + } + const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); + expect(models[0].entities[0].fields.map(f => f.name)).toEqual(['f']); + expect( + warnings.some(w => /missing its name; the field is skipped/i.test(w))) + .toBe(true); + }); }); @@ -452,6 +481,37 @@ describe('metric attach entity is re-derived from the expression', () => { expect(byName.get('revenue')!.entity).toBe('orders'); expect(byName.get('mix')!.entity).toBeUndefined(); }); + + test( + 'a cross-entity metric falls back to the persisted attach entity', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + // Two entities in the expression, so it cannot be re-derived; the + // authored attach entity must ride back on the persisted aspect. + metrics: [{ + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)', + entity: 'orders', + }], + }; + const {models} = roundTrip(model); + expect(models[0].metrics[0].entity).toBe('orders'); + }); }); From 8768a40dda29ee339591ec628568bb4f1db112d3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 01:33:02 +0000 Subject: [PATCH 10/12] mdcode: reconcile pull with the merged --emit-expressions gating PR #278 landed on main gating the KC SQL-expression fields off by default: the emitter now omits the per-field `semantics` block (field expression + role) and the metric expression unless `--emit-expressions` is set, and the committed emitter golden was regenerated without them. The pull leg was built against the old always-emit behavior, so after rebasing onto main it read back less than its tests and goldens assumed. Reconcile the reader to the new default: - readMetric no longer warns when a metric aspect has no expression -- that is now the expected default, not a malformed aspect. It still derives the attach entity from the expression when one is present, else from the persisted `entity`, and still warns on an expression that pins no known entity. - Reader header + user guide: field/metric expressions and the DIMENSION role are recovered only from an `--emit-expressions` push; a default push -> pull drops them. - Tests: split the round trip into roundTrip (default, drops the semantics block) and roundTripFull (`--emit-expressions`, keeps it); point the lossless-slice, DIMENSION, canonical-expression, and expression-derivation cases at the full round trip, and add a case pinning the default drop. The symmetry floor now strips field/metric expressions and the dimension role. - Regenerate the .pull.golden.yaml fixtures: they drop only expressions and bare dimension markers, matching the default emitter golden. --- toolbox/mdcode/docs/semantic-model.md | 15 ++- .../mdcode/src/libts/semantic/kc_converter.ts | 32 ++++--- .../sales_bq_graph_target.pull.golden.yaml | 12 --- .../star_orders_customer.pull.golden.yaml | 33 ------- .../fixtures/tpcds_date_edge.pull.golden.yaml | 94 ------------------- .../tests/libts/semantic/kc_converter.test.ts | 78 +++++++++++---- .../tests/libts/semantic/pull_kc.test.ts | 8 +- 7 files changed, 95 insertions(+), 177 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b3a1fcbb..b4aaa106 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -266,17 +266,24 @@ with no matching catalog entry) is left untouched — pull never deletes. > > **Recovered exactly** — these come back as authored: > - Model structure: the model, its entities, and each entity's fields. -> - Field data source and data type; the field expression. -> - Metrics: name, expression, data type, and attach entity. +> - Field data source and data type. +> - Metrics: name, data type, and attach entity. > - 1:1 / 1:N relationships: endpoints, foreign-key direction, and join columns > (from the `schema-join` links). > - Deployment targets. > +> **Recovered only if pushed with `--emit-expressions`** — the per-field +> `semantics` block (expressions and the dimension role) and the metric +> expression are omitted from the catalog by default (see the note above), so +> pull returns them only when the push that wrote them used `--emit-expressions`: +> - Field expressions and metric expressions (the canonical GoogleSQL/ANSI form). +> - A field's dimension role, which comes back as a bare `dimension: {}` marker, +> without its detail (`is_time`, and so on). A default push drops the marker +> entirely. +> > **Recovered, but normalized** — the content survives, the form changes: > - Relationship *names* come back lowercased/hyphenated (the catalog stores the > name only in the link id, e.g. `Places Order` → `places-order`). -> - A field marked as a dimension comes back as a bare `dimension: {}` marker, -> without its detail (`is_time`, and so on). > - A metric authored with no data type comes back as an explicit `Decimal` > (push must write a type, and defaults it to `NUMERIC`). > diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index e732ad55..9c863290 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -26,15 +26,20 @@ // // Fidelity is bounded by what the emitter persisted, so this read is the // inverse of the WRITE, not of the authored document. It recovers names, -// descriptions, data sources, field datatypes (via the schema aspect) and -// DIMENSION roles, field/metric expressions, each metric's attach entity -// (re-derived from its expression, as the loader does, else the value the -// emitter persisted), the model's deployment targets (from the semantic-model -// aspect, back into the GOOGLE `custom_extensions` block), and 1:1 / 1:N -// relationships (from the `schema-join` entry links a pull fetched -- see -// `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what -// the emitter does not write: entity keys/unique keys, `ai_context`, field -// labels, `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// descriptions, data sources, field datatypes (via the schema aspect), each +// metric's attach entity (re-derived from its expression, as the loader does, +// when the catalog holds one, else the value the emitter persisted), the +// model's deployment targets (from the semantic-model aspect, back into the +// GOOGLE `custom_extensions` block), and 1:1 / 1:N relationships (from the +// `schema-join` entry links a pull fetched -- see +// `modelsFromCatalogResources`'s `entryLinks` argument). The per-field +// `semantics` block -- field/metric expressions and the DIMENSION role -- is +// gated off the catalog by default: the emitter writes it only under +// `--emit-expressions` (see `KcGenerateOptions.emitExpressions`), so those +// three recover only when the push that wrote them enabled it, and a default +// push -> pull drops them. It cannot recover what the emitter never writes: +// entity keys/unique keys, `ai_context`, field labels, +// `importedExpression`/`importedDialect` (the vendor-dialect SQL), and // many-to-many (association) relationships (whose edge lives only in the // BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads // back un-typed: the metric aspect persists only `dataType` (both collapse to @@ -197,15 +202,14 @@ function readField(fd: any, entityName: string, warnings: string[]): Field| // Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` -// is re-derived from the expression (as the loader does) rather than read from -// the aspect, so it stays consistent with the reconstructed entity set. +// is re-derived from the expression (as the loader does) when the aspect holds +// one, else it falls back to the entity the emitter persisted. A default push +// omits the expression entirely (it is gated behind `--emit-expressions`), so +// an absent expression is expected, not an error, and does not warn. function readMetric( entry: Entry, entityNames: string[], warnings: string[]): Metric { const name = entry.entrySource?.displayName ?? idOf(entry.name); const data = aspectData(entry, 'semantic-metric'); - if (data.expression === undefined) { - warnings.push(`metric '${name}': no expression in semantic-metric aspect`); - } const metric: Metric = {name}; if (data.expression !== undefined) metric.expression = data.expression; diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml index 06dd8bdc..f966f619 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -10,19 +10,7 @@ semantic_model: source: demo.sales.orders fields: - name: o_orderkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderkey - name: o_totalprice - expression: - dialects: - - dialect: BIGQUERY - expression: o_totalprice metrics: - name: total_revenue - expression: - dialects: - - dialect: BIGQUERY - expression: SUM(orders.o_totalprice) datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml index d7a9cbe1..ceb6bed1 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -12,40 +12,15 @@ semantic_model: description: One row per order fields: - name: o_orderkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderkey description: Order identifier - name: o_custkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_custkey - name: o_orderdate - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderdate - dimension: {} - name: o_totalprice - expression: - dialects: - - dialect: BIGQUERY - expression: o_totalprice - name: customer source: samples.tpch.customer fields: - name: c_custkey - expression: - dialects: - - dialect: BIGQUERY - expression: c_custkey - name: c_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_name description: Customer name relationships: - name: orders-to-customer @@ -57,16 +32,8 @@ semantic_model: - c_custkey metrics: - name: total_revenue - expression: - dialects: - - dialect: BIGQUERY - expression: SUM(orders.o_totalprice) datatype: Decimal description: Total order revenue - name: order_count - expression: - dialects: - - dialect: BIGQUERY - expression: COUNT(orders.o_orderkey) datatype: Decimal description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml index 1900fe12..ab45bace 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -9,138 +9,44 @@ semantic_model: description: Fact table containing all store sales transactions fields: - name: ss_item_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_item_sk - dimension: {} - name: ss_ticket_number - expression: - dialects: - - dialect: BIGQUERY - expression: ss_ticket_number - dimension: {} - name: ss_customer_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_customer_sk - dimension: {} - name: ss_store_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_store_sk - dimension: {} - name: ss_quantity - expression: - dialects: - - dialect: BIGQUERY - expression: ss_quantity description: Quantity of items sold - name: ss_sales_price - expression: - dialects: - - dialect: BIGQUERY - expression: ss_sales_price description: Sales price per unit - name: ss_ext_sales_price - expression: - dialects: - - dialect: BIGQUERY - expression: ss_ext_sales_price description: Extended sales price (quantity * price) - name: ss_net_profit - expression: - dialects: - - dialect: BIGQUERY - expression: ss_net_profit description: Net profit from the sale - name: customer source: tpcds.public.customer description: Customer dimension with demographic information fields: - name: c_customer_sk - expression: - dialects: - - dialect: BIGQUERY - expression: c_customer_sk - dimension: {} - name: c_first_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_first_name - dimension: {} description: Customer first name - name: c_last_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_last_name - dimension: {} description: Customer last name - name: item source: tpcds.public.item description: Item/Product dimension fields: - name: i_item_sk - expression: - dialects: - - dialect: BIGQUERY - expression: i_item_sk - dimension: {} - name: i_brand - expression: - dialects: - - dialect: BIGQUERY - expression: i_brand - dimension: {} - name: i_category - expression: - dialects: - - dialect: BIGQUERY - expression: i_category - dimension: {} - name: i_current_price - expression: - dialects: - - dialect: BIGQUERY - expression: i_current_price description: Current price of the item - name: store source: tpcds.public.store description: Store dimension with location attributes fields: - name: s_store_sk - expression: - dialects: - - dialect: BIGQUERY - expression: s_store_sk - dimension: {} - name: s_store_name - expression: - dialects: - - dialect: BIGQUERY - expression: s_store_name - dimension: {} - name: s_city - expression: - dialects: - - dialect: BIGQUERY - expression: s_city - dimension: {} - name: s_state - expression: - dialects: - - dialect: BIGQUERY - expression: s_state - dimension: {} - name: s_number_employees - expression: - dialects: - - dialect: BIGQUERY - expression: s_number_employees description: Number of employees at the store - name: date_dim source: sqlgen-testing.demo.date_dim diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index e63d6bf0..4ccac618 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -7,13 +7,16 @@ // emitter is lossless. The write drops content by design (entity keys, // ai_context, field labels, importedDialect, and many-to-many relationships -- // see the emitter header), so the expected read-back is the source model with -// exactly those fields cleared. 1:1 / 1:N relationships and deployment targets -// DO round-trip (via schema-join links and the semantic-model aspect), except -// that relationship names come back normalized (lowercased/hyphenated). -// Targeted tests pin the mapping details a round trip cannot isolate (the -// dataType inverse, the DIMENSION role, resource-URI parsing, metric attach -// re-derivation, relationship endpoint/direction recovery, and parent/anchor -// grouping). +// exactly those fields cleared. It ALSO drops the per-field `semantics` block +// (field/metric expressions and the DIMENSION role) unless the push enabled +// `emitExpressions`, so a DEFAULT round trip (roundTrip) loses those too, while +// a full round trip (roundTripFull, emitExpressions: true) keeps them. 1:1 / +// 1:N relationships and deployment targets DO round-trip either way (via +// schema-join links and the semantic-model aspect), except that relationship +// names come back normalized (lowercased/hyphenated). Targeted tests pin the +// mapping details a round trip cannot isolate (the dataType inverse, the +// DIMENSION role, resource-URI parsing, metric attach re-derivation, +// relationship endpoint/direction recovery, and parent/anchor grouping). import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; @@ -33,18 +36,32 @@ const OPTS = { entryGroup: 'eg' }; -// Emits a model to entries + entry links and reads it straight back. +// Emits a model to entries + entry links and reads it straight back. The +// default push omits the per-field `semantics` block (expressions + role), so a +// default round trip drops those; use roundTripFull to exercise their recovery. function roundTrip(model: SemanticModel): {models: SemanticModel[]; warnings: string[]} { const {entries, entryLinks} = generateCatalogResources(model, OPTS); return modelsFromCatalogResources(entries, entryLinks); } +// A round trip through a `--emit-expressions` push: the catalog then holds the +// per-field `semantics` block and the metric expression, so field/metric +// expressions and DIMENSION roles round-trip too. +function roundTripFull(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries, entryLinks} = + generateCatalogResources(model, {...OPTS, emitExpressions: true}); + return modelsFromCatalogResources(entries, entryLinks); +} + describe('emitter -> reader round trip (lossless slice)', () => { // A model using only round-trippable content: no keys/ai_context/labels/ // relationships (all dropped by the write), and datatypes that invert - // cleanly. + // cleanly. Its fields and metric carry expressions and a DIMENSION role, so + // it round-trips losslessly only through a `--emit-expressions` push + // (roundTripFull); a default push omits the per-field `semantics` block. const source: SemanticModel = { name: 'sales', description: 'the sales model', @@ -73,7 +90,7 @@ describe('emitter -> reader round trip (lossless slice)', () => { }; test('reconstructs an IR equal to the source', () => { - const {models} = roundTrip(source); + const {models} = roundTripFull(source); expect(models).toHaveLength(1); expect(models[0]).toEqual(source); }); @@ -357,30 +374,46 @@ describe('dataType inverse (schema aspect -> IR type)', () => { describe('field mapping details', () => { - function readField(field: Entity['fields'][number]) { + function readWith( + rt: (m: SemanticModel) => {models: SemanticModel[]}, + field: Entity['fields'][number]) { const model: SemanticModel = { name: 'm', entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: [field]}], relationships: [], metrics: [], }; - return roundTrip(model).models[0].entities[0].fields[0]; + return rt(model).models[0].entities[0].fields[0]; } + // Default push (no per-field `semantics`) vs a `--emit-expressions` push. + const readField = (f: Entity['fields'][number]) => readWith(roundTrip, f); + const readFieldFull = (f: Entity['fields'][number]) => + readWith(roundTripFull, f); test('a DIMENSION role reads back as a dimension marker', () => { - const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + // The role lives in the gated `semantics` block, so it survives only a + // `--emit-expressions` push. + const back = readFieldFull({name: 'd', expression: 'e.d', dimension: {}}); expect(back.dimension).toEqual({}); }); test('a non-dimension field has no dimension marker', () => { - const back = readField({name: 'f', expression: 'e.f'}); + const back = readFieldFull({name: 'f', expression: 'e.f'}); expect(back.dimension).toBeUndefined(); }); + test( + 'a default push drops the DIMENSION role with the semantics block', + () => { + const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toBeUndefined(); + expect(back.expression).toBeUndefined(); + }); + test( 'an imported expression is not persisted to or recovered from the catalog', () => { - const back = readField({ + const back = readFieldFull({ name: 'amt', expression: 'e.amt', importedExpression: 'e.amt::NUMBER', @@ -476,7 +509,9 @@ describe('metric attach entity is re-derived from the expression', () => { }, ], }; - const {models} = roundTrip(model); + // Full push: the expression is in the catalog, so the entity is + // re-derived from it (single entity -> attach; two entities -> none). + const {models} = roundTripFull(model); const byName = new Map(models[0].metrics.map(m => [m.name, m])); expect(byName.get('revenue')!.entity).toBe('orders'); expect(byName.get('mix')!.entity).toBeUndefined(); @@ -559,7 +594,9 @@ describe('metric expression referencing no known entity', () => { // References `widgets`, which is not an entity of this model. metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], }; - const {models, warnings} = roundTrip(model); + // The unplaceable warning fires from the expression, so it needs a push + // that wrote one (a default push omits it, leaving nothing to check). + const {models, warnings} = roundTripFull(model); expect(models[0].metrics[0].entity).toBeUndefined(); expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) .toBe(true); @@ -688,7 +725,11 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { delete f.importedExpression; // vendor SQL is not persisted delete f.importedDialect; delete f.customExtensions; - if (f.dimension) f.dimension = {}; // only the DIMENSION role survives + // A default push omits the per-field `semantics` block, so the field + // expression and the DIMENSION role are not persisted (they ride back + // only through a `--emit-expressions` push). + delete f.expression; + delete f.dimension; // String is indistinguishable from an un-typed field on read. if (f.type === 'String') delete f.type; } @@ -699,6 +740,7 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { delete metric.importedExpression; delete metric.importedDialect; delete metric.customExtensions; + delete metric.expression; // omitted by a default push, like field ones // The emitter writes a required dataType, defaulting typeless -> NUMERIC, // which reads back as Decimal; String collapses to un-typed like fields. if (metric.type === undefined) { diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index 2316003d..83f5aee0 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -47,9 +47,13 @@ const SALES: SemanticModel = { }], }; -// The entries the emitter would have written for a model. +// The entries the emitter would have written for a model. Emitted with +// expressions on (a `--emit-expressions` push) so a model that carries field / +// metric expressions reconstructs exactly; the default push omits them (the +// converter tests in kc_converter.test.ts pin that gating). function entriesFor(model: SemanticModel): Entry[] { - return generateCatalogResources(model, OPTS).entries; + return generateCatalogResources(model, {...OPTS, emitExpressions: true}) + .entries; } function ok(result?: T): ApiResult { From b36b7585d15a597ce47526383786b44934ef7433 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 02:34:24 +0000 Subject: [PATCH 11/12] mdcode: document pull round-trip writer-side follow-ups Distinguish inherent pull losses from write-side limits: relationship names (not stored in the schema-join aspect) and non-canonical deployment targets (dropped on write) could round-trip faithfully with a writer change. Reader already recovers everything the catalog holds. --- toolbox/mdcode/docs/semantic-model.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b4aaa106..21b3e5f4 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -298,6 +298,22 @@ with no matching catalog entry) is left untouched — pull never deletes. > pulled document as a faithful copy of the catalog metadata, not of the authored > model, and keep the authored document as the source of truth. +> **Note — writer-side follow-ups (not inherent to pull).** Two of the reductions +> above are limits of what push currently *writes*, not of what pull can recover. +> They are recorded here as write-side follow-ups; the reader (pull) already +> returns everything the catalog holds. +> +> - **Relationship names.** The `schema-join` aspect does not store the authored +> relationship name, so pull recovers it from the link id — which is lowercased +> and hyphenated. Persisting the name in the aspect on write would let pull +> return it verbatim. +> - **Non-canonical deployment targets.** Push persists a target only when it is a +> canonical BigQuery Graph URL +> (`//bigquery.googleapis.com/projects/.../datasets/.../propertyGraphs/...`). +> Other forms — a misspelled path, or the `projects/.../entryGroups/@bigquery/` +> entry form — are dropped on write, so pull has nothing to recover. Widening or +> normalizing the writer's accepted forms would let them round-trip. + ## Permissions `push` needs access to whichever destinations you deploy to. From abc370b1f29f25743be3c9b4663b1276ff7d96b6 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 03:48:54 +0000 Subject: [PATCH 12/12] mdcode: document malformed deployment-target rejection A non-canonical BigQuery Graph deployment target fails push at the validation gate, before any leg and for every --target, so nothing is written to BigQuery or Knowledge Catalog. Correct the earlier writer-side follow-up note, which wrongly implied such targets are silently dropped on write, and sharpen the relationship-name follow-up to name the server-side schema-join aspect-type template as the fix. --- toolbox/mdcode/docs/semantic-model.md | 41 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 21b3e5f4..b111776f 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -180,7 +180,16 @@ paired columns and foreign-key direction — in its aspect. `push` and `--validate-only` run the same checks, **before either destination is touched**, so a model that cannot deploy fails fast instead of half-deploying: -* **Exactly one deployment target per model.** *(static)* +* **Exactly one deployment target per model, and it must be a valid BigQuery + Graph URI.** A model with no target — or with more than one — is rejected, and + so is a single target whose URI does not match + `//bigquery.googleapis.com/projects/

/datasets//propertyGraphs/` (for + example a `propertyGraph`/`propertyGraphs` typo, or a + `…/entryGroups/@bigquery/entries/…` entry form). The error names the offending + URI and the expected form. This gate runs before any destination leg and for + every `--target`, so a malformed target writes **nothing** — not to BigQuery + and **not to Knowledge Catalog**; the push aborts with a non-zero exit and no + entries are created. *(static)* * **Every metric on a BigQuery Graph model resolves to exactly one entity** — otherwise it would be dropped from the BigQuery Graph. Set the metric's attach entity, or scope its expression to a single entity. *(static)* @@ -298,21 +307,23 @@ with no matching catalog entry) is left untouched — pull never deletes. > pulled document as a faithful copy of the catalog metadata, not of the authored > model, and keep the authored document as the source of truth. -> **Note — writer-side follow-ups (not inherent to pull).** Two of the reductions -> above are limits of what push currently *writes*, not of what pull can recover. -> They are recorded here as write-side follow-ups; the reader (pull) already -> returns everything the catalog holds. +> **Note — writer-side follow-up (not inherent to pull).** One reduction above is +> a limit of what push currently *writes*, not of what pull can recover. It is +> recorded here as a write-side follow-up; the reader (pull) already returns +> everything the catalog holds. > -> - **Relationship names.** The `schema-join` aspect does not store the authored -> relationship name, so pull recovers it from the link id — which is lowercased -> and hyphenated. Persisting the name in the aspect on write would let pull -> return it verbatim. -> - **Non-canonical deployment targets.** Push persists a target only when it is a -> canonical BigQuery Graph URL -> (`//bigquery.googleapis.com/projects/.../datasets/.../propertyGraphs/...`). -> Other forms — a misspelled path, or the `projects/.../entryGroups/@bigquery/` -> entry form — are dropped on write, so pull has nothing to recover. Widening or -> normalizing the writer's accepted forms would let them round-trip. +> - **Relationship names.** The `schema-join` aspect type's `metadataTemplate` has +> no field for the relationship name, so push cannot store it and pull recovers +> it from the link id — which is lowercased and hyphenated (the entry-link id +> format forbids the original casing/underscores). Returning the name verbatim +> requires adding a name field to the built-in `schema-join` aspect type in +> Knowledge Catalog (server-side), after which the client write/read is trivial; +> it is the same class of gap as the `semantics` field that gates +> `--emit-expressions`. +> +> (A non-canonical deployment target is **not** a pull gap: push rejects it at the +> validation gate before any leg runs, so it is never written — see +> [Validation](#validation).) ## Permissions