From d8a4c633100bd39045a04afe21cd58278581bdb3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 04:44:39 +0000 Subject: [PATCH 01/14] mdcode: add semantic-model pull from Knowledge Catalog Add the inverse of the KC emitter: read semantic-model / -entity / -metric entries and their aspects back into the IR, serialize to YAML, and wire a 'pull' command (with --dry-run and --model) for the semantic-model scope. Re-stacked onto the KC-push follow-ups: the emitter no longer writes importedExpression, so the reader no longer recovers it; idOf is shared from knowledge_catalog.ts; and push entry/link writes use the same bounded mapConcurrent pool as pull hydration. --- .../src/libts/layouts/semantic-model.ts | 30 ++ .../semantic/deploy_knowledge_catalog.ts | 199 +++++++++++- .../src/libts/semantic/knowledge_catalog.ts | 263 +++++++++++++++- .../mdcode/src/libts/semantic/serialize.ts | 256 ++++++++++++++++ toolbox/mdcode/src/tool/commands.ts | 90 +++++- toolbox/mdcode/src/tool/main.ts | 6 +- .../deploy_knowledge_catalog.pull.test.ts | 244 +++++++++++++++ .../semantic/knowledge_catalog.read.test.ts | 258 ++++++++++++++++ .../semantic/semantic_model_layout.test.ts | 75 +++++ .../tests/libts/semantic/serialize.test.ts | 290 ++++++++++++++++++ 10 files changed, 1695 insertions(+), 16 deletions(-) create mode 100644 toolbox/mdcode/src/libts/semantic/serialize.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/serialize.test.ts diff --git a/toolbox/mdcode/src/libts/layouts/semantic-model.ts b/toolbox/mdcode/src/libts/layouts/semantic-model.ts index 214fe930..42730af7 100644 --- a/toolbox/mdcode/src/libts/layouts/semantic-model.ts +++ b/toolbox/mdcode/src/libts/layouts/semantic-model.ts @@ -91,6 +91,36 @@ export class SemanticModelLayout implements CatalogLayout { return docs; } + // True when a model document with this handle already exists on disk. `pull` + // uses it to report which files it would overwrite vs. create. + hasModel(name: string): boolean { + return fs.existsSync(this.modelPath(name)); + } + + // The absolute path a model document with this handle maps to: + // `/EntryGroups//.yaml`. Path separators in the + // model name are replaced so a name still yields a single flat file. Requires + // the layout to be scoped to an entry group (the semantic-model source always + // is). + modelPath(name: string): string { + if (!this._entryGroup) { + throw new Error( + 'SemanticModel layout has no entry group; cannot resolve a model path.'); + } + const file = `${name.replace(/[/\\]/g, '_')}.yaml`; + return path.join(this._catalogPath, 'EntryGroups', this._entryGroup, file); + } + + // Writes a model's serialized document to its path, creating the EntryGroup + // directory if needed, and indexes it so a later modelDocuments() sees it. + // This is the sink `pull` writes reconstructed models to. + writeModelDocument(name: string, text: string): void { + const localPath = this.modelPath(name); + fs.mkdirSync(path.dirname(localPath), {recursive: true}); + fs.writeFileSync(localPath, text); + this._index.set(name, localPath); + } + // The Knowledge Catalog entry-level members are not applicable to this // push-only layout; the model is authored as a single Ossie document, not as // per-entry Knowledge Catalog files. These are wired when KC-resource emit diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 5676ec86..0bf69527 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -38,7 +38,8 @@ import {ApiResult} from '../gcp/api'; import * as context from '../gcp/context'; import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; -import {generateCatalogResources, KcResources} from './knowledge_catalog'; +import {SemanticModel} from './ir'; +import {generateCatalogResources, idOf, KcResources, modelsFromCatalogResources} from './knowledge_catalog'; import {LoadedModel} from './loader'; @@ -120,6 +121,10 @@ interface Counts { type EmittedModel = {model: string; resources: KcResources}; +// Upper bound on concurrent entry / entry-link writes within one model's wave +// (mirrors HYDRATE_CONCURRENCY on the pull side). +const WRITE_CONCURRENCY = 8; + // entries.create propagation retry: a just-created entry group can briefly 404. const ENTRY_CREATE_TRIES = 3; const ENTRY_CREATE_RETRY_MS = 3000; @@ -566,7 +571,8 @@ async function createEntries( const isMetric = (e: Entry) => (e.entryType ?? '').endsWith('/semantic-metric'); for (const wave of [children.filter(e => !isMetric(e)), children.filter(isMetric)]) { - const res = await Promise.all(wave.map(e => writeEntry(cat, opts, e))); + const res = await mapConcurrent( + wave, WRITE_CONCURRENCY, e => writeEntry(cat, opts, e)); const firstErr = res.find(r => r.error); if (firstErr) return {created, updated, error: firstErr.error}; for (const r of res) { @@ -593,7 +599,8 @@ async function createEntryLinks( cat: CatalogClient, opts: KcDeployOptions, links: EntryLink[]): Promise { if (!links.length) return {linked: 0}; - const res = await Promise.all(links.map(l => writeEntryLink(cat, opts, l))); + const res = await mapConcurrent( + links, WRITE_CONCURRENCY, l => writeEntryLink(cat, opts, l)); const firstErr = res.find(r => r.error); if (firstErr) return {linked: 0, error: firstErr.error}; return {linked: links.length}; @@ -686,11 +693,6 @@ function planSummary( } -// The id segment of a full entry/entryType resource name (after the last '/'). -function idOf(name: string): string { - return name.split('/').pop() ?? name; -} - function isOk(res: {status: number}): boolean { return res.status === 200; } @@ -715,3 +717,184 @@ function isPropagating(res: {message?: string}): boolean { function errText(res: {status: number; message?: string}): string { return res.message?.trim() || `HTTP ${res.status}`; } + + +// --------------------------------------------------------------------------- +// Pull: Knowledge Catalog -> Semantic Model IR. +// +// The read counterpart of deployKnowledgeCatalog and the inverse of push. +// Unlike a write, a pull needs no server-side type provisioning -- only that +// the `semantic-*` entries exist. It enumerates the entry group, keeps the +// semantic entries, hydrates each one's aspect data (a BASIC list omits aspect +// data, so each entry is re-fetched with its aspect types -- an entity needs +// BOTH its `semantic-entity` aspect and the built-in `schema` aspect), and +// hands the hydrated entries to the pure reader (modelsFromCatalogResources). +// --------------------------------------------------------------------------- + +export interface KcPullOptions { + project: string; + location: string; + entryGroup: string; + model?: string; // limit to a single model by name (default: all) +} + +export interface KcPullResult { + models: SemanticModel[]; + warnings: string[]; +} + +// Upper bound on in-flight aspect-hydration fetches during a pull. +const HYDRATE_CONCURRENCY = 8; + +// Reads the semantic models back from a Knowledge Catalog entry group. Emits no +// console output; warnings (skipped entries, no match for --model, reader +// warnings) are returned for the caller to print. +export async function pullKnowledgeCatalog( + cat: CatalogClient, opts: KcPullOptions): Promise { + const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const warnings: string[] = []; + + // Enumerate the group (paging is inherently sequential) and pick the semantic + // entries, then hydrate their aspects concurrently: a BASIC list omits aspect + // data, so each entry needs its own lookupEntry, and those fetches are + // independent. The pool preserves input order so warnings stay deterministic. + const targets: {entry: Entry; aspectTypes: string[]}[] = []; + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const aspectTypes = semanticAspectTypes(entry.entryType); + if (aspectTypes) targets.push({entry, aspectTypes}); + // else: not part of a semantic model; ignore it. + } + + // When scoped to one model, hydrate only that model's entries -- its anchor + // (matched by name) plus the children pointing at it. A list already carries + // entrySource + parentEntry, so this avoids fetching every other model's + // aspects. No match short-circuits with just the not-found warning. + let scoped = targets; + if (opts.model) { + scoped = scopeToModel(targets, opts.model); + if (!scoped.length) { + return { + models: [], + warnings: [ + `no semantic model named '${opts.model}' found in ${destination}` + ], + }; + } + } + + const fetched = await mapConcurrent( + scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry '${entry.name}' (status ${ + res.status}); skipped` + }; + } + return {entry: res.result}; + }); + + const hydrated: Entry[] = []; + for (const r of fetched) { + if (r.entry) + hydrated.push(r.entry); + else if (r.warning) + warnings.push(r.warning); + } + + const read = modelsFromCatalogResources(hydrated); + warnings.push(...read.warnings); + + // Defense in depth: keep only the requested model even if the reader surfaced + // another anchor (e.g. a child whose parentEntry pointed outside the scope). + let models = read.models; + if (opts.model) { + models = models.filter(m => m.name === opts.model); + if (!models.length) { + warnings.push( + `no semantic model named '${opts.model}' found in ${destination}`); + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// The aspect type resource names to hydrate for a semantic entry, derived from +// its entryType (the aspect types are the parallel resources in the same +// project/location). An entity carries two aspects: its `semantic-entity` +// aspect and the built-in `schema` aspect that holds its fields. Returns +// undefined for entries that are not part of a semantic model. +function semanticAspectTypes(entryType: string): string[]|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + const typeBase = entryType.slice(0, idx); + const t = entryType.slice(idx + marker.length); + const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + switch (t) { + case 'semantic-model': + return [aspectType('semantic-model')]; + case 'semantic-entity': + return [aspectType('semantic-entity'), aspectType('schema')]; + case 'semantic-metric': + return [aspectType('semantic-metric')]; + default: + return undefined; + } +} + + +// Restricts hydration targets to a single model: the semantic-model anchor +// whose name (entrySource.displayName, else the entry id) matches `model`, plus +// every child entry whose parentEntry is that anchor. Uses only list-level +// fields (no aspect data), so it runs before hydration and avoids fetching +// unrelated models' aspects. Returns [] when no anchor matches. +function scopeToModel( + targets: {entry: Entry; aspectTypes: string[]}[], + model: string): {entry: Entry; aspectTypes: string[]}[] { + const isAnchor = (t: {entry: Entry}) => + !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); + const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = new Set( + targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === + model) + .map(t => t.entry.name)); + if (!matchedAnchorNames.size) return []; + // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds + // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a + // project-id normalization mismatch) still belongs to it. Without this a + // scoped pull would drop children a full pull keeps. + const soleAnchor = + allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; + const soleMatched = + soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); + return targets.filter( + t => matchedAnchorNames.has(t.entry.name) || + matchedAnchorNames.has(t.entry.parentEntry ?? '') || + (soleMatched && !isAnchor(t) && + !allAnchorNames.has(t.entry.parentEntry ?? ''))); +} + + +// Maps `items` through `fn` with at most `limit` calls in flight, returning +// results in input order (so downstream ordering stays deterministic). +async function mapConcurrent( + items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker(): Promise { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + } + const workers = + Array.from({length: Math.min(limit, items.length)}, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index a3051b22..5e5860c9 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -45,7 +45,8 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {bigQueryGraphTargets} from './deploy_bigquery'; -import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; +import {DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; +import {referencedEntityNames} from './sql_expr_utils'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -539,3 +540,263 @@ function linkSlug(s: string): string { function unquote(part: string): string { return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); } + + +// --------------------------------------------------------------------------- +// Reader: Knowledge Catalog entries -> Semantic Model IR. +// +// The inverse of generateCatalogResources: it reconstructs the IR from the +// entries a pull hydrated (see deploy_knowledge_catalog.pullKnowledgeCatalog). +// `semantic-entity` / `semantic-metric` entries are grouped under their +// `semantic-model` anchor via `parentEntry`; entries of other types are +// ignored. Resources are matched by type-name SUFFIX, so a reader need not know +// which system-type project/location the emitter used. +// +// Fidelity is bounded by what the emitter persisted, so this read is the +// inverse of the WRITE, not of the authored document. It recovers names, +// descriptions, data sources, field datatypes (via the schema aspect) and +// DIMENSION roles, field/metric expressions, and each +// metric's attach entity (re-derived from its expression, as the loader does). +// It cannot recover what the emitter does not write: entity keys/unique keys, +// `ai_context`, field labels, `importedDialect`, `custom_extensions`, and +// relationships (the graph edges live in the BigQuery property graph, not the +// catalog). +// --------------------------------------------------------------------------- + +export interface ReadResult { + models: SemanticModel[]; + warnings: string[]; +} + +/** + * Reconstructs the Semantic Model IR from Knowledge Catalog entries. + * + * Returns one model per `semantic-model` anchor plus any warnings (no anchor, + * an orphaned child, an entry missing its aspect data). Entries must already be + * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a + * BASIC list omits aspect data, so the puller re-fetches each entry first. + */ +export function modelsFromCatalogResources(entries: Entry[]): ReadResult { + const warnings: string[] = []; + + const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); + const entityEntries = + entries.filter(e => semanticType(e) === 'semantic-entity'); + const metricEntries = + entries.filter(e => semanticType(e) === 'semantic-metric'); + + if (!anchors.length) { + warnings.push('no semantic-model entry found; nothing to reconstruct'); + return {models: [], warnings: [...new Set(warnings)]}; + } + + // A child belongs to its anchor by parentEntry. When there is exactly one + // anchor, children whose parentEntry does not resolve (e.g. a project-id + // normalization mismatch) are still attached to it rather than dropped. + const anchorNames = new Set(anchors.map(a => a.name)); + const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; + const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( + e => e.parentEntry === anchorName || + (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + + const models = anchors.map(anchor => { + const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + + const entities = childrenOf(anchor.name, entityEntries) + .map(e => readEntity(e, warnings)); + const entityNames = entities.map(e => e.name); + const metrics = childrenOf(anchor.name, metricEntries) + .map(e => readMetric(e, entityNames, warnings)); + + // Relationships are not published to the catalog (see the file header), so + // a reconstructed model always has an empty edge set. + const model: SemanticModel = {name, entities, relationships: [], metrics}; + const description = anchor.entrySource?.description; + if (description !== undefined) model.description = description; + return model; + }); + + // Flag children that resolved to no anchor at all (only possible with + // multiple anchors, where the sole-anchor fallback does not apply). + if (!soleAnchor) { + for (const child of [...entityEntries, ...metricEntries]) { + if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { + warnings.push(`entry '${ + child.name}' has no resolvable parent semantic-model; omitted`); + } + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// Reconstructs an entity from its `semantic-entity` aspect (the backing source) +// and the built-in `schema` aspect (its fields). Keys are not persisted by the +// emitter and so come back empty. +function readEntity(entry: Entry, warnings: string[]): Entity { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const semantic = aspectData(entry, 'semantic-entity'); + const schema = aspectData(entry, 'schema'); + if (!Object.keys(semantic).length) { + warnings.push(`entity '${ + name}': no semantic-entity aspect data (fetch with the aspect type)`); + } + + const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); + if (!dataSource) { + warnings.push( + `entity '${name}': no backing data source in the semantic-entity ` + + `aspect; 'source' will be empty and the entity may not load`); + } + const entity: Entity = { + name, + dataSource, + keys: [], // not persisted by the emitter; unrecoverable on read + fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + }; + const description = entry.entrySource?.description; + if (description !== undefined) entity.description = description; + return entity; +} + + +// Reconstructs a field from one `schema` aspect field record, inverting +// schemaAspectData: the datatype from dataType/metadataType, expressions from +// the nested `semantics` block, and the DIMENSION role back to a dimension +// marker. +function readField(fd: any, entityName: string, warnings: string[]): Field { + const name = fd?.name; + if (name === undefined || name === '') { + warnings.push( + `entity '${entityName}': a schema field is missing its name; the ` + + `field may not load`); + } + const field: Field = {name}; + const sem = fd?.semantics ?? {}; + if (sem.expression !== undefined) field.expression = sem.expression; + const type = irDataType(fd?.dataType, fd?.metadataType); + if (type !== undefined) field.type = type; + if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.description !== undefined) field.description = fd.description; + return field; +} + + +// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` +// is re-derived from the expression (as the loader does) rather than read from +// the aspect, so it stays consistent with the reconstructed entity set. +function readMetric( + entry: Entry, entityNames: string[], warnings: string[]): Metric { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + if (data.expression === undefined) { + warnings.push(`metric '${name}': no expression in semantic-metric aspect`); + } + + const metric: Metric = {name}; + if (data.expression !== undefined) metric.expression = data.expression; + const exprForRefs = data.expression ?? ''; + const referenced = referencedEntityNames(exprForRefs, entityNames); + if (referenced.length === 1) { + metric.entity = referenced[0]; + } else if (exprForRefs && !referenced.length) { + // Parity with the loader's convertMetric: an expression that qualifies no + // known entity is flagged as potentially unplaceable downstream. + warnings.push( + `metric '${name}': expression references no known entity; it may not ` + + `be placeable downstream`); + } + // The emitter writes a required dataType, defaulting a typeless metric to + // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric + // authored without a datatype round-trips as an explicit Decimal rather than + // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) + const type = irDataType(data.dataType, undefined); + if (type !== undefined) metric.type = type; + const description = entry.entrySource?.description; + if (description !== undefined) metric.description = description; + return metric; +} + + +// The inverse of columnDataType/columnMetadataType: maps the schema aspect's +// dataType (disambiguated by metadataType only for the STRING family) back to +// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read +// as un-typed (undefined) -- the loader's default -- since the emitter cannot +// distinguish an authored `String` from an un-typed field (both emit STRING). +function irDataType(dataType: string|undefined, metadataType: string|undefined): + DataType|undefined { + switch (dataType) { + case 'INT64': + return 'Integer'; + case 'NUMERIC': + return 'Decimal'; + case 'FLOAT64': + return 'Float'; + case 'BOOL': + return 'Boolean'; + case 'DATE': + return 'Date'; + case 'TIME': + return 'Time'; + case 'DATETIME': + return 'DateTime'; + case 'TIMESTAMP': + return 'DateTimeTz'; + case 'STRING': + return metadataType === 'OTHER' ? 'Opaque' : undefined; + default: + return undefined; + } +} + + +// The inverse of resourcePath: a BigQuery linked-resource URI becomes the +// canonical `project.dataset.table` string; anything else (a verbatim query or +// a passthrough reference) is returned unchanged. +function dataSourceFromResource(resource: string|undefined): string { + const value = (resource ?? '').trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + + +// The bare `semantic-*` type of an entry, matched by entryType suffix so the +// system-type project/location need not be known. Returns undefined for entries +// that are not part of a semantic model. +function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| + 'semantic-metric'|undefined { + for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as + const) { + if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; + } + return undefined; +} + + +// The `data` payload of an entry's aspect of the given bare type, matched by +// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` +// suffix (robust to whichever system-type project/location the emitter used). +// Returns an empty object when the aspect is absent. +function aspectData(entry: Entry, type: string): Record { + const aspects = entry.aspects ?? {}; + for (const [key, aspect] of Object.entries(aspects)) { + if (key.endsWith(`.${type}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + + +// The id segment of a full entry resource name (after the last '/'). +export function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} diff --git a/toolbox/mdcode/src/libts/semantic/serialize.ts b/toolbox/mdcode/src/libts/semantic/serialize.ts new file mode 100644 index 00000000..dd215100 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/serialize.ts @@ -0,0 +1,256 @@ +// Serializes the Semantic Model IR (./ir) back to the open AI-first semantics +// format (YAML). This is the inverse of `loader.ts`: `loader` reads authored +// YAML into the IR; this module writes the IR out as a YAML document the loader +// can read back. It is the local-workspace sink for `pull` (Knowledge Catalog +// -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR -> a +// destination. +// +// Fidelity is at the IR level, not byte-for-byte with a hand-authored file. The +// loader normalizes several authoring conveniences into the IR at load time, so +// they are already gone before serialization and cannot be reproduced here: +// * per-dialect `expression.dialects[]` variants collapse to at most two +// forms +// (a target/canonical `expression` + an `importedExpression`); only those +// are re-emitted, each under a single dialect label. +// * comments and key ordering are not preserved. +// What the loader DOES keep on the IR round-trips here: names, descriptions, +// `ai_context` (instructions / synonyms / examples), `custom_extensions` +// (verbatim), keys and unique keys, data sources, field datatypes / labels / +// dimension flags, expressions, and relationship join columns. See +// serialize.test.ts. +// +// An `association` (junction-table) relationship has no open-format syntax (the +// loader cannot produce one), so only its direct foreign-key view (from/to + +// columns) is serialized; the junction detail is dropped with a note. + +import * as yaml from 'yaml'; + +import {AiContext, CustomExtension, Entity, Field, Metric, Relationship, SemanticModel,} from './ir'; + +// The schema version the loader was written against; re-emitted verbatim so a +// serialized document loads without a version-mismatch warning. Mirrors +// loader.SUPPORTED_VERSION. +const SERIALIZED_VERSION = '0.2.0.dev0'; + +// The dialect label for the IR's target/canonical `expression`. The IR does not +// record which authored dialect that string came from (the loader picked it +// from the target dialect or the ANSI_SQL fallback and discarded the label), +// and its contract is "GoogleSQL-valid". BIGQUERY is the loader's default +// target dialect, so labeling the canonical form BIGQUERY makes the loader +// re-pick it exactly on reload -- a clean round trip with no dialect-fallback +// note. +const CANONICAL_DIALECT = 'BIGQUERY'; + +// The dialect label used for an `importedExpression` whose `importedDialect` +// was lost (e.g. read back from a Knowledge Catalog aspect that does not +// persist the dialect). A non-target, non-canonical label so the loader +// re-reads it as the imported vendor form rather than the canonical expression. +const UNKNOWN_IMPORTED_DIALECT = 'IMPORTED'; + +export interface SerializeResult { + yaml: string; + warnings: string[]; +} + +/** + * Serializes a single semantic model to a YAML document string in the open + * AI-first semantics format. The document contains exactly this one model; + * `pull` writes one file per model + * (catalog/EntryGroups//.yaml). + * + * Warnings flag IR content that has no loadable representation (an association + * relationship's junction detail), so the caller can surface the lossy edge. + */ +export function serializeModel(model: SemanticModel): SerializeResult { + const warnings: string[] = []; + const text = yaml.stringify(modelDocument(model, warnings)); + return {yaml: text, warnings: [...new Set(warnings)]}; +} + +// Builds the plain document object (version + one model) that yaml.stringify +// renders. Kept separate so tests can assert the structure without parsing +// YAML. +export function modelDocument( + model: SemanticModel, warnings: string[] = []): Record { + return { + version: SERIALIZED_VERSION, + semantic_model: [modelDoc(model, warnings)], + }; +} + +function modelDoc( + model: SemanticModel, warnings: string[]): Record { + // `datasets` is required (min 1) by the loader. A reconstructed model with no + // entities (e.g. every entity fetch failed during a pull) would serialize to + // a document the loader rejects; emit the (empty) array but flag it so the + // lossy edge is visible rather than surfacing later as an opaque load error. + const datasets = (model.entities ?? []).map(e => datasetDoc(e, warnings)); + if (!datasets.length) { + warnings.push( + `model '${model.name}': no datasets (entities); the document requires ` + + `at least one and will not load until an entity is present.`); + } + return compact({ + name: model.name, + description: model.description, + ai_context: aiContextDoc(model.aiContext), + custom_extensions: customExtensionsDoc(model.customExtensions), + datasets, + relationships: nonEmpty( + (model.relationships ?? []).map(r => relationshipDoc(r, warnings))), + metrics: nonEmpty((model.metrics ?? []).map(m => metricDoc(m, warnings))), + }); +} + +function datasetDoc( + entity: Entity, warnings: string[]): Record { + return compact({ + name: entity.name, + source: entity.dataSource, + primary_key: nonEmpty(entity.keys), + unique_keys: nonEmpty(entity.uniqueKeys), + description: entity.description, + ai_context: aiContextDoc(entity.aiContext), + fields: nonEmpty((entity.fields ?? []).map(f => fieldDoc(f, warnings))), + custom_extensions: customExtensionsDoc(entity.customExtensions), + }); +} + +function fieldDoc(field: Field, warnings: string[]): Record { + const expression = expressionDoc( + field.expression, field.importedExpression, field.importedDialect); + if (!expression) { + warnings.push( + `field '${field.name}': no expression; the loader requires one per ` + + `field and the document will not load until it is set.`); + } + return compact({ + name: field.name, + expression, + datatype: field.type, + label: field.label, + dimension: dimensionDoc(field), + description: field.description, + ai_context: aiContextDoc(field.aiContext), + custom_extensions: customExtensionsDoc(field.customExtensions), + }); +} + +function metricDoc(metric: Metric, warnings: string[]): Record { + // `entity` is derived by the loader from the expression's entity qualifiers, + // so it is intentionally not emitted: the loader recomputes it on reload. + const expression = expressionDoc( + metric.expression, metric.importedExpression, metric.importedDialect); + if (!expression) { + warnings.push( + `metric '${metric.name}': no expression; the loader requires one per ` + + `metric and the document will not load until it is set.`); + } + return compact({ + name: metric.name, + expression, + datatype: metric.type, + description: metric.description, + ai_context: aiContextDoc(metric.aiContext), + custom_extensions: customExtensionsDoc(metric.customExtensions), + }); +} + +// Inverts loader.convertRelationship: `from`/`to` are the endpoint entities and +// `from_columns`/`to_columns` are their positional join columns. An association +// (junction-table) edge has no open-format syntax, so only this direct-FK view +// is emitted and the junction detail is flagged. +function relationshipDoc( + rel: Relationship, warnings: string[]): Record { + if (rel.association) { + warnings.push( + `relationship '${ + rel.name}': association (junction-table) detail has no ` + + `open-format representation and is not serialized; only its foreign-key ` + + `endpoints are written.`); + } + return compact({ + name: rel.name, + from: rel.source.entity, + to: rel.destination.entity, + from_columns: nonEmpty(rel.source.columns), + to_columns: nonEmpty(rel.destination.columns), + description: rel.description, + ai_context: aiContextDoc(rel.aiContext), + custom_extensions: customExtensionsDoc(rel.customExtensions), + }); +} + +// Renders the `expression` object matching the loader's schema (a `dialects` +// array of {dialect, expression}). Emits the target/canonical form under +// CANONICAL_DIALECT and the imported vendor form under its own dialect, so the +// loader re-picks each into the same IR field. Returns undefined when neither +// form is present (the loader requires an expression, but a pathological field +// with none is dropped rather than fabricated). +function expressionDoc( + expression: string|undefined, importedExpression: string|undefined, + importedDialect: string|undefined): Record|undefined { + const dialects: {dialect: string; expression: string}[] = []; + if (expression !== undefined) { + dialects.push({dialect: CANONICAL_DIALECT, expression}); + } + if (importedExpression !== undefined) { + let label = importedDialect ?? UNKNOWN_IMPORTED_DIALECT; + // Never emit two dialect entries under the same label: the loader would + // pick between them non-deterministically. If the imported form's dialect + // collides with the canonical label already pushed, fall back to the + // imported placeholder. + if (expression !== undefined && label === CANONICAL_DIALECT) { + label = UNKNOWN_IMPORTED_DIALECT; + } + dialects.push({dialect: label, expression: importedExpression}); + } + return dialects.length ? {dialects} : undefined; +} + +// Emits the field's `dimension` block when present, inverting +// loader.convertField (which sets `dimension = {}` for a bare marker and copies +// `is_time`). The key is emitted whenever the IR carries dimension metadata, +// even for an empty marker, so a dimension field reloads as a dimension. +function dimensionDoc(field: Field): Record|undefined { + if (!field.dimension) return undefined; + return compact({is_time: field.dimension.isTime}); +} + +// Emits the structured `ai_context` (the only authoring path the loader reads +// synonyms/instructions/examples from), inverting loader.normalizeAiContext. +// Returns undefined when the context carries nothing, so `compact` drops the +// key. +function aiContextDoc(ai: AiContext|undefined): Record|undefined { + if (!ai) return undefined; + const doc = compact({ + instructions: ai.instructions, + synonyms: nonEmpty(ai.synonyms), + examples: nonEmpty(ai.examples), + }); + return Object.keys(doc).length ? doc : undefined; +} + +// Emits vendor `custom_extensions` verbatim (`vendorName` -> `vendor_name`), +// inverting loader.toCustomExtensions. Returns undefined when there are none. +function customExtensionsDoc(exts: CustomExtension[]|undefined): + Record[]|undefined { + if (!exts || !exts.length) return undefined; + return exts.map(e => ({vendor_name: e.vendorName, data: e.data})); +} + +// Drops undefined-valued keys so the emitted YAML only shows fields the model +// actually set (matching the emitters' `compact` convention). +function compact>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +// Returns the array, or undefined when empty/absent, so an empty list is +// omitted rather than rendered as `[]`. +function nonEmpty(items: T[]|undefined): T[]|undefined { + return items && items.length ? items : undefined; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index bd8cf696..59ce84e1 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -14,6 +14,7 @@ import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; import {BigQueryClient} from '../libts/gcp/bigquery'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; +import { serializeModel } from '../libts/semantic/serialize'; import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; @@ -158,15 +159,20 @@ export async function init(options: InitOptions): Promise { } -export async function pull(): Promise { +export interface PullOptions { + // Reconstruct + report only; never writes a file. Mirrors push --validate-only. + dryRun?: boolean; + // Limit the pull to a single model by name (default: all in the entry group). + model?: string; +} + + +export async function pull(options: PullOptions = {}): Promise { const ctx = context.ApiContext.default(); const snapshot = await kcmd.CatalogSnapshot.fromPath('.', ctx); if (snapshot.manifest.source.type === Sources.SEMANTIC_MODEL) { - console.log( - 'Semantic-model scope: nothing to pull. Knowledge Catalog resource ' + - 'pull for the semantic model is not yet implemented.'); - return 0; + return await pullSemanticModel(ctx, snapshot, options); } const catalog = new dataplex.CatalogClient(ctx); @@ -360,3 +366,77 @@ async function pushKnowledgeCatalog( unlinked}.`); return 0; } + + +// Pulls the semantic model's Knowledge Catalog entries back into local model +// documents (catalog/EntryGroups//.yaml) and prints the +// result. The destination coordinates come from the scope +// (project.location.entryGroup). Overwrite policy matches the core pull: +// last-write-wins, local-only documents are left untouched (never deleted). +// Returns a process exit code (0 on success). +async function pullSemanticModel( + ctx: context.ApiContext, snapshot: kcmd.CatalogSnapshot, + options: PullOptions): Promise { + // The semantic-model source always resolves to the SemanticModel layout + // (see createLayout), so these casts are safe. + const layout = snapshot.layout as SemanticModelLayout; + const source = snapshot.manifest.source as SemanticModelSource; + + console.log(options.dryRun + ? 'Reconstructing semantic model from Knowledge Catalog (dry run)...' + : 'Pulling semantic model from Knowledge Catalog...'); + + const catalog = new dataplex.CatalogClient(ctx); + const result = await kc.pullKnowledgeCatalog(catalog, { + project: source.project, + location: source.location, + entryGroup: source.entryGroup, + model: options.model, + }); + + for (const w of result.warnings) { + console.warn(`Warning: ${w}`); + } + + if (!result.models.length) { + console.log('No semantic models found; nothing to pull.'); + return 0; + } + + let created = 0; + let updated = 0; + // Guard against two reconstructed models whose names map to the same file + // (path-separator sanitizing, or two anchors sharing a display name): the + // later write would silently clobber the earlier. Track written paths so the + // collision is reported and the dry-run/real counts agree on the repeat. + const writtenBy = new Map(); + for (const model of result.models) { + const serialized = serializeModel(model); + for (const w of serialized.warnings) { + console.warn(`Warning: [${model.name}] ${w}`); + } + const target = layout.modelPath(model.name); + const prior = writtenBy.get(target); + if (prior !== undefined && prior !== model.name) { + console.warn( + `Warning: models '${prior}' and '${model.name}' both map to ` + + `${target}; the later overwrites the earlier -- rename one model.`); + } + const existed = writtenBy.has(target) || layout.hasModel(model.name); + writtenBy.set(target, model.name); + if (options.dryRun) { + console.log(` would ${existed ? 'update' : 'create'} ${target}`); + } + else { + layout.writeModelDocument(model.name, serialized.yaml); + console.log(` ${existed ? 'updated' : 'created'} ${target}`); + } + if (existed) updated++; + else created++; + } + + console.log(options.dryRun + ? `Dry run: would write ${created} new and ${updated} updated model document(s).` + : `Wrote ${created} new and ${updated} updated model document(s).`); + return 0; +} diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index cbc67e95..84cceff9 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -25,10 +25,12 @@ cli.command('init', 'Initialize a new catalog snapshot') cli.command('pull', 'Pull catalog entries') - .action(async () => { + .option('--dry-run', 'Reconstruct and report only; do not write files (semantic-model scope)') + .option('--model ', 'Limit the pull to a single model by name (semantic-model scope)') + .action(async (options) => { let exitCode = 1; try { - exitCode = await commands.pull(); + exitCode = await commands.pull(options); } catch (err: any) { console.error('Error:', err.message || err); diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts new file mode 100644 index 00000000..e31a2f94 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts @@ -0,0 +1,244 @@ +// Tests for the semantic-model Knowledge Catalog pull leg +// (pullKnowledgeCatalog in src/libts/semantic/deploy_knowledge_catalog.ts). +// +// pullKnowledgeCatalog is the orchestration around the pure reader: enumerate +// the entry group, hydrate each semantic entry's aspect data, and reconstruct +// the IR. The catalog client is stubbed so no network call is made. The entries +// the fake serves are produced by the real emitter, so this exercises the true +// list -> hydrate -> read path end to end (an entity is re-fetched with BOTH +// its semantic-entity and schema aspects). The reader's own mapping is covered +// in knowledge_catalog.read.test.ts; the focus here is the fetch SEQUENCE: +// aspect hydration, the --model filter, skipped entries, and ignoring foreign +// entries. + +import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; + +import {ApiResult} from '../../../src/libts/gcp/api'; +import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +const SALES: SemanticModel = { + name: 'sales', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [ + {name: 'o_totalprice', expression: 'orders.o_totalprice', type: 'Decimal'} + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + // Metrics carry a datatype through KC (semantic-metric.dataType is + // required); Decimal -> NUMERIC on emit, NUMERIC -> Decimal on read. + type: 'Decimal' + }], +}; + +// The entries the emitter would have written for a model. +function entriesFor(model: SemanticModel): Entry[] { + return generateCatalogResources(model, OPTS).entries; +} + +function ok(result?: T): ApiResult { + return {status: 200, result}; +} +function err(status: number, message: string): ApiResult { + return {status, message}; +} + +// Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, +// or 404s an unknown name). `lookupFail` forces a failure for one entry name. +function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): + {list: any; lookup: any} { + const byName = new Map(served.map(e => [e.name, e])); + const list = spyOn(CatalogClient.prototype, 'listEntries') + .mockImplementation(async function*() { + for (const e of listed) yield e; + } as any); + const lookup = spyOn(CatalogClient.prototype, 'lookupEntry') + .mockImplementation(async (_p, _l, name) => { + if (name === lookupFail) return err(500, 'boom'); + const e = byName.get(name); + return e ? ok(e) : err(404, 'not found'); + }); + return {list, lookup}; +} + +afterEach(() => { + mock.restore(); +}); + + +describe('pullKnowledgeCatalog: happy path', () => { + test('reconstructs the model from listed + hydrated entries', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models).toHaveLength(1); + expect(models[0]).toEqual(SALES); + expect(warnings).toHaveLength(0); + // Every listed semantic entry (model + entity + metric) was hydrated. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + 'an entity is hydrated with BOTH its semantic-entity and schema aspects', + async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + await pullKnowledgeCatalog(cat, OPTS); + + // Find the lookup call for the entity entry and inspect its requested + // aspect types (the 4th arg). + const entityEntry = + entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + const call = + lookup.mock.calls.find((c: any[]) => c[2] === entityEntry.name)!; + const aspectTypes: string[] = call[3]; + expect( + aspectTypes.some(t => t.endsWith('/aspectTypes/semantic-entity'))) + .toBe(true); + expect(aspectTypes.some(t => t.endsWith('/aspectTypes/schema'))) + .toBe(true); + }); +}); + + +describe('pullKnowledgeCatalog: filtering and robustness', () => { + test('--model keeps only the named model', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + }); + + test('--model with no match returns nothing and warns', async () => { + const entries = entriesFor(SALES); + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); + + test( + 'a non-semantic entry in the group is ignored, not fetched', async () => { + const entries = entriesFor(SALES); + const foreign: Entry = { + name: `projects/dest/locations/us/entryGroups/eg/entries/foreign`, + entryType: 'projects/x/locations/us/entryTypes/some-other-type', + }; + const {lookup} = stubClient([...entries, foreign], entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + expect(models).toHaveLength(1); + // The foreign entry is never hydrated (only the 3 semantic entries + // are). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('a failed hydration is skipped with a warning', async () => { + const entries = entriesFor(SALES); + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + stubClient(entries, entries, metricEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + // The model + entity still reconstruct; only the metric is dropped. + expect(models).toHaveLength(1); + expect(models[0].metrics).toHaveLength(0); + expect(warnings.some( + w => /failed to fetch/i.test(w) && w.includes(metricEntry.name))) + .toBe(true); + }); + + test('--model hydrates only the target model\'s entries', async () => { + const other: SemanticModel = { + name: 'inventory', + entities: + [{name: 'items', dataSource: 'p.d.items', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + const entries = [...entriesFor(SALES), ...entriesFor(other)]; + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + expect(models.map(m => m.name)).toEqual(['sales']); + // SALES has 3 entries (model + entity + metric); inventory's are never + // fetched -- the flag scopes hydration, not just the final result. + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test( + '--model keeps a child whose parentEntry does not resolve (sole-anchor ' + + 'fallback)', + async () => { + const entries = entriesFor(SALES); + // Simulate a project-id normalization mismatch: the metric points at an + // anchor name that differs from the emitted one. With a single model in + // the group a full pull still attaches it via the reader's sole-anchor + // fallback, so a scoped pull must keep it too (else --model silently + // drops a child a full pull returns). + const metricEntry = + entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + metricEntry.parentEntry = + metricEntry.parentEntry!.replace('projects/dest/', 'projects/12345/'); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'sales'}); + + expect(models).toHaveLength(1); + expect(models[0].metrics.map(m => m.name)).toEqual(['total_revenue']); + // The metric was hydrated despite the parent mismatch (3 entries). + expect(lookup).toHaveBeenCalledTimes(3); + }); + + test('--model with no match fetches nothing and warns', async () => { + const entries = entriesFor(SALES); + const {lookup} = stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models, warnings} = + await pullKnowledgeCatalog(cat, {...OPTS, model: 'nope'}); + expect(models).toHaveLength(0); + expect(lookup).not.toHaveBeenCalled(); + expect(warnings.some(w => /no semantic model named 'nope'/i.test(w))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts new file mode 100644 index 00000000..a8d5e447 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts @@ -0,0 +1,258 @@ +// Behavior specification for the Knowledge Catalog reader +// (modelsFromCatalogResources in src/libts/semantic/knowledge_catalog.ts). +// +// The reader is the inverse of the emitter (generateCatalogResources). The +// central guarantee is an emitter -> reader round trip: emit a model's entries, +// read them back, and get an IR equal to the source WHERE the emitter is +// lossless. The write drops content by design (entity keys, ai_context, field +// labels, importedDialect, relationships -- see the emitter header), so the +// expected read-back is the source model with exactly those fields cleared. +// Targeted tests pin the mapping details a round trip cannot isolate (the +// dataType inverse, the DIMENSION role, resource-URI parsing, metric attach +// re-derivation, and parent/anchor grouping). + +import {describe, expect, test} from 'bun:test'; + +import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {generateCatalogResources, modelsFromCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; + +const OPTS = { + project: 'dest', + location: 'us', + entryGroup: 'eg' +}; + +// Emits a model to entries and reads it straight back. +function roundTrip(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources(entries); +} + + +describe('emitter -> reader round trip (lossless slice)', () => { + // A model using only round-trippable content: no keys/ai_context/labels/ + // relationships (all dropped by the write), and datatypes that invert + // cleanly. + const source: SemanticModel = { + name: 'sales', + description: 'the sales model', + entities: [{ + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], // keys are not persisted; keep empty so the round trip matches + fields: [ + {name: 'o_orderkey', expression: 'orders.o_orderkey', type: 'Integer'}, + { + name: 'o_orderdate', + expression: 'orders.o_orderdate', + type: 'Date', + dimension: {}, + description: 'order date', + }, + ], + }], + relationships: [], + metrics: [{ + name: 'total_revenue', + expression: 'SUM(orders.o_totalprice)', + entity: 'orders', + type: 'Decimal', + }], + }; + + test('reconstructs an IR equal to the source', () => { + const {models} = roundTrip(source); + expect(models).toHaveLength(1); + expect(models[0]).toEqual(source); + }); +}); + + +describe('dataType inverse (schema aspect -> IR type)', () => { + // Emit a one-field model of each IR type, read it back, and check the field's + // reconstructed type. String and Opaque both emit dataType STRING; String + // (indistinguishable from an un-typed field) reads back as undefined, while + // Opaque is disambiguated by metadataType OTHER. + const cases: [Metric['type']|undefined, Metric['type']|undefined][] = [ + ['Integer', 'Integer'], + ['Decimal', 'Decimal'], + ['Float', 'Float'], + ['Boolean', 'Boolean'], + ['Date', 'Date'], + ['Time', 'Time'], + ['DateTime', 'DateTime'], + ['DateTimeTz', 'DateTimeTz'], + ['Opaque', 'Opaque'], + ['String', undefined], // collapses to un-typed + [undefined, undefined], // un-typed stays un-typed + ]; + + for (const [type, expected] of cases) { + test(`${type ?? 'un-typed'} -> ${expected ?? 'un-typed'}`, () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f', ...(type ? {type} : {})}], + }], + relationships: [], + metrics: [], + }; + const back = roundTrip(model).models[0].entities[0].fields[0]; + expect(back.type).toBe(expected as any); + }); + } +}); + + +describe('field mapping details', () => { + function readField(field: Entity['fields'][number]) { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: [field]}], + relationships: [], + metrics: [], + }; + return roundTrip(model).models[0].entities[0].fields[0]; + } + + test('a DIMENSION role reads back as a dimension marker', () => { + const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toEqual({}); + }); + + test('a non-dimension field has no dimension marker', () => { + const back = readField({name: 'f', expression: 'e.f'}); + expect(back.dimension).toBeUndefined(); + }); + + test( + 'an imported expression is not persisted to or recovered from the catalog', + () => { + const back = readField({ + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }); + expect(back.expression).toBe('e.amt'); + // The emitter no longer writes importedExpression/importedDialect, so + // neither survives the round trip. + expect(back.importedExpression).toBeUndefined(); + expect(back.importedDialect).toBeUndefined(); + }); +}); + + +describe('data source resource-path parsing', () => { + function readDataSource(dataSource: string): string { + const model: SemanticModel = { + name: 'm', + entities: [{name: 'e', dataSource, keys: [], fields: []}], + relationships: [], + metrics: [], + }; + return roundTrip(model).models[0].entities[0].dataSource; + } + + test( + 'a three-part BigQuery reference round-trips through the resource URI', + () => { + expect(readDataSource('proj.ds.tbl')).toBe('proj.ds.tbl'); + }); + + test('a verbatim query source is preserved unchanged', () => { + const query = 'SELECT * FROM t'; + expect(readDataSource(query)).toBe(query); + }); +}); + + +describe('metric attach entity is re-derived from the expression', () => { + test( + 'a single-entity metric attaches; a cross-entity metric does not', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + metrics: [ + {name: 'revenue', expression: 'SUM(orders.amt)', entity: 'orders'}, + { + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)' + }, + ], + }; + const {models} = roundTrip(model); + const byName = new Map(models[0].metrics.map(m => [m.name, m])); + expect(byName.get('revenue')!.entity).toBe('orders'); + expect(byName.get('mix')!.entity).toBeUndefined(); + }); +}); + + +describe('anchor / parent grouping', () => { + test('no semantic-model entry yields no models and a warning', () => { + const {models, warnings} = modelsFromCatalogResources([]); + expect(models).toHaveLength(0); + expect(warnings.some(w => /no semantic-model entry/i.test(w))).toBe(true); + }); + + test('two models keep their own children by parentEntry', () => { + const a = generateCatalogResources( + { + name: 'a', + entities: [{name: 'ea', dataSource: 'p.d.a', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const b = generateCatalogResources( + { + name: 'b', + entities: [{name: 'eb', dataSource: 'p.d.b', keys: [], fields: []}], + relationships: [], + metrics: [], + }, + OPTS); + const {models} = modelsFromCatalogResources([...a.entries, ...b.entries]); + const byName = new Map(models.map(m => [m.name, m])); + expect(byName.get('a')!.entities.map(e => e.name)).toEqual(['ea']); + expect(byName.get('b')!.entities.map(e => e.name)).toEqual(['eb']); + }); +}); + + +describe('metric expression referencing no known entity', () => { + test('warns that the metric may be unplaceable and leaves it unattached', + () => { + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], + relationships: [], + // References `widgets`, which is not an entity of this model. + metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].metrics[0].entity).toBeUndefined(); + expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) + .toBe(true); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts new file mode 100644 index 00000000..ac4c7bc0 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/semantic_model_layout.test.ts @@ -0,0 +1,75 @@ +// Tests for the SemanticModel layout's pull write-path +// (src/libts/layouts/semantic-model.ts): modelPath / hasModel / +// writeModelDocument. These are the sink `pull` writes reconstructed models to; +// the push-side discovery (modelDocuments) is exercised via the deploy tests. + +import {afterEach, beforeEach, describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; + +import {SemanticModelLayout} from '../../../src/libts/layouts/semantic-model'; + +let root: string; +let catalogPath: string; + +beforeEach(() => { + root = fs.mkdtempSync(path.join(os.tmpdir(), 'kcmd-layout-')); + catalogPath = path.join(root, 'catalog'); +}); + +afterEach(() => { + fs.rmSync(root, {recursive: true, force: true}); +}); + +async function layout(entryGroup?: string): Promise { + const l = new SemanticModelLayout(catalogPath, entryGroup); + await l.init(); + return l; +} + + +describe('SemanticModelLayout write path', () => { + test('modelPath maps to EntryGroups//.yaml', async () => { + const l = await layout('eg'); + expect(l.modelPath('sales')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'sales.yaml')); + }); + + test('modelPath sanitizes path separators in the model name', async () => { + const l = await layout('eg'); + expect(l.modelPath('a/b')) + .toBe(path.join(catalogPath, 'EntryGroups', 'eg', 'a_b.yaml')); + }); + + test('modelPath throws without an entry group', async () => { + const l = await layout(undefined); + expect(() => l.modelPath('sales')).toThrow(/entry group/i); + }); + + test( + 'writeModelDocument creates the file, dirs, and indexes it', async () => { + const l = await layout('eg'); + expect(l.hasModel('sales')).toBe(false); + + l.writeModelDocument('sales', 'version: x\n'); + + const p = l.modelPath('sales'); + expect(fs.existsSync(p)).toBe(true); + expect(fs.readFileSync(p, 'utf8')).toBe('version: x\n'); + expect(l.hasModel('sales')).toBe(true); + // Indexed, so a subsequent read surfaces it as a model document. + expect(l.modelDocuments()).toEqual([ + {name: 'sales', text: 'version: x\n'} + ]); + }); + + test( + 'writeModelDocument overwrites an existing document (last-write-wins)', + async () => { + const l = await layout('eg'); + l.writeModelDocument('sales', 'first\n'); + l.writeModelDocument('sales', 'second\n'); + expect(fs.readFileSync(l.modelPath('sales'), 'utf8')).toBe('second\n'); + }); +}); diff --git a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts b/toolbox/mdcode/tests/libts/semantic/serialize.test.ts new file mode 100644 index 00000000..81c4454c --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/serialize.test.ts @@ -0,0 +1,290 @@ +// Behavior specification for the semantic-model serializer +// (src/libts/semantic/serialize.ts). +// +// serialize.ts is the inverse of loader.ts: IR -> open-format YAML. The +// strongest guarantee is a round trip through the loader -- load a fixture to +// the IR, serialize it, load the serialized text again, and assert the two IRs +// are identical. That pins IR-level fidelity across every feature the loader +// produces (datasets, fields, datatypes, dimensions, labels, ai_context, +// custom_extensions, relationships, metrics, imported expressions) without +// hard-coding YAML text. Targeted structural tests cover the mapping details a +// round trip cannot isolate. + +import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as yaml from 'yaml'; + +import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; +import {loadModels} from '../../../src/libts/semantic/loader'; +import {modelDocument, serializeModel} from '../../../src/libts/semantic/serialize'; + +const FIXTURES = path.join(__dirname, 'fixtures'); + +function loadFixture(name: string): SemanticModel[] { + const text = fs.readFileSync(path.join(FIXTURES, name), 'utf8'); + return loadModels(text).models; +} + + +describe('loader <-> serialize round trip is IR-stable', () => { + // Each fixture exercises a different slice of the format: relationships + + // ai_context + synonyms + label + dimension; the full tpc-ds corpus with + // custom_extensions + unique_keys; explicit datatypes; and imported + // (vendor-dialect) expressions. + const fixtures = [ + 'star_orders_customer.yaml', + 'tpcds_retail.yaml', + 'sales_google_ext.yaml', + 'vendor_dialects.yaml', + 'lineitem_databricks_ext.yaml', + 'sales_bq_graph_target.yaml', + ]; + + for (const fixture of fixtures) { + test(`${fixture} survives IR -> YAML -> IR unchanged`, () => { + const original = loadFixture(fixture); + expect(original.length).toBeGreaterThan(0); + + for (const model of original) { + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models; + expect(reloaded).toHaveLength(1); + // IR-level equality: every field the loader keeps must match exactly. + expect(reloaded[0]).toEqual(model); + } + }); + } +}); + + +describe('serialized document structure', () => { + const model = loadFixture('star_orders_customer.yaml')[0]; + const doc = modelDocument(model) as any; + const sm = doc.semantic_model[0]; + + test('emits the supported version and a single model', () => { + expect(doc.version).toBe('0.2.0.dev0'); + expect(doc.semantic_model).toHaveLength(1); + expect(sm.name).toBe(model.name); + }); + + test('a dataset source is the opaque dataSource string, verbatim', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.source).toBe(entity.dataSource); + expect(typeof orders.source).toBe('string'); + }); + + test('primary_key mirrors the entity keys', () => { + const orders = sm.datasets.find((d: any) => d.name === 'orders'); + const entity = model.entities.find(e => e.name === 'orders')!; + expect(orders.primary_key).toEqual(entity.keys); + }); + + test('ai_context is emitted structurally (synonyms round-trip)', () => { + // The fixture annotates a field (o_orderdate) with synonyms; find it and + // assert the structured ai_context is emitted under that field. + const entity = model.entities.find( + e => e.fields.some(f => f.aiContext?.synonyms?.length))!; + const field = entity.fields.find(f => f.aiContext?.synonyms?.length)!; + const dsDoc = sm.datasets.find((d: any) => d.name === entity.name); + const fieldDoc = dsDoc.fields.find((f: any) => f.name === field.name); + expect(fieldDoc.ai_context.synonyms).toEqual(field.aiContext!.synonyms); + }); + + test('a relationship maps to from/to + positional columns', () => { + expect(sm.relationships.length).toBeGreaterThan(0); + const rel = model.relationships[0]; + const relDoc = sm.relationships[0]; + expect(relDoc.from).toBe(rel.source.entity); + expect(relDoc.to).toBe(rel.destination.entity); + expect(relDoc.from_columns).toEqual(rel.source.columns); + expect(relDoc.to_columns).toEqual(rel.destination.columns); + }); +}); + + +describe('expression + datatype + dimension mapping', () => { + test('an explicit datatype round-trips as `datatype`', () => { + const model = loadFixture('sales_google_ext.yaml')[0]; + const typed = model.entities.flatMap(e => e.fields).find(f => f.type); + expect(typed).toBeDefined(); + const {yaml: text} = serializeModel(model); + const reloaded = loadModels(text).models[0]; + const back = reloaded.entities.flatMap(e => e.fields) + .find(f => f.name === typed!.name)!; + expect(back.type).toBe(typed!.type); + }); + + test('a bare dimension marker survives as `dimension: {}`', () => { + const field: + Field = {name: 'ship_date', expression: 'e.ship_date', dimension: {}}; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const fieldDoc = doc.semantic_model[0].datasets[0].fields[0]; + expect(fieldDoc.dimension).toEqual({}); + // And it reloads back to a dimension field. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.entities[0].fields[0].dimension).toEqual({}); + }); + + test('an imported vendor expression is emitted under its own dialect', () => { + const field: Field = { + name: 'amt', + expression: 'e.amt', + importedExpression: 'e.amt::NUMBER', + importedDialect: 'SNOWFLAKE', + }; + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: ['k'], fields: [field]}], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const dialects = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects; + const labels = dialects.map((d: any) => d.dialect); + expect(labels).toContain('SNOWFLAKE'); + // The canonical form is labeled BIGQUERY so the loader re-picks it exactly. + expect(labels).toContain('BIGQUERY'); + }); + + test('a metric does not emit its derived attach entity', () => { + const metric: Metric = { + name: 'total', + expression: 'SUM(orders.amt)', + entity: 'orders', + }; + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{name: 'amt', expression: 'orders.amt'}], + }], + relationships: [], + metrics: [metric], + }; + const metricDoc = + (modelDocument(model) as any).semantic_model[0].metrics[0]; + expect(metricDoc).not.toHaveProperty('entity'); + // The loader re-derives it on reload. + const reloaded = loadModels(serializeModel(model).yaml).models[0]; + expect(reloaded.metrics[0].entity).toBe('orders'); + }); +}); + + +describe('lossy edges are flagged', () => { + test( + 'an association relationship warns and drops the junction detail', () => { + const rel: Relationship = { + name: 'enrollment', + source: {entity: 'student', columns: ['id']}, + destination: {entity: 'course', columns: ['id']}, + association: { + dataSource: 'p.d.enrollment', + keys: ['student_id', 'course_id'], + sourceColumns: ['student_id'], + destinationColumns: ['course_id'], + }, + }; + const model: SemanticModel = { + name: 'school', + entities: [ + { + name: 'student', + dataSource: 'p.d.student', + keys: ['id'], + fields: [] + }, + { + name: 'course', + dataSource: 'p.d.course', + keys: ['id'], + fields: [] + }, + ], + relationships: [rel], + metrics: [], + }; + const {yaml: text, warnings} = serializeModel(model); + expect(warnings.some(w => /association/i.test(w))).toBe(true); + // The direct-FK view is still emitted (from/to + columns), so it + // reloads. + const relDoc = yaml.parse(text).semantic_model[0].relationships[0]; + expect(relDoc.from).toBe('student'); + expect(relDoc.to).toBe('course'); + expect(relDoc.from_columns).toEqual(['id']); + }); +}); + + +describe('serialize flags loader-invalid reconstructions', () => { + test('a model with no entities warns that datasets is required', () => { + const model: SemanticModel = { + name: 'empty', + entities: [], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /no datasets/i.test(w))).toBe(true); + }); + + test('a field with no expression warns that one is required', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'orphan'}], + }], + relationships: [], + metrics: [], + }; + const {warnings} = serializeModel(model); + expect(warnings.some(w => /field 'orphan'.*no expression/i.test(w))) + .toBe(true); + }); + + test( + 'an imported dialect colliding with the canonical label is relabeled', + () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'orders', + dataSource: 'p.d.t', + keys: [], + fields: [{ + name: 'amt', + expression: 'orders.amt', + importedExpression: 'orders.AMT', + importedDialect: 'BIGQUERY', + }], + }], + relationships: [], + metrics: [], + }; + const doc = modelDocument(model) as any; + const labels: string[] = + doc.semantic_model[0].datasets[0].fields[0].expression.dialects.map( + (d: any) => d.dialect); + // No duplicate dialect label: the imported form is relabeled off + // BIGQUERY so the loader does not pick between two BIGQUERY entries. + expect(new Set(labels).size).toBe(labels.length); + expect(labels).toContain('BIGQUERY'); + }); +}); From 2d7f9bf407609514e4c93d6d308d49f1dba1310e Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:22:44 +0000 Subject: [PATCH 02/14] mdcode: restructure pull into converter-scaffold files; add fixture goldens + docs Addresses PR review feedback on the KC pull leg: - Rename serialize.ts -> osi_converter.ts (the OSI <-> IR converter). The name now says what it converts between; a header banner notes it currently holds only the serialize direction and that the loader migrates in post-#278. - Extract the KC reader into kc_converter.ts and the network pull into pull_kc.ts, so the new capability lives in its own files rather than swelling knowledge_catalog.ts / deploy_knowledge_catalog.ts. Those two files return to their #278 state (emit-only / push-only). This is scaffolding for the eventual two-layer split (pure converters vs push/pull orchestration); the remaining halves move in once #278 merges, with no further file renames. - Reorganize the pull tests around committed golden artifacts: each corpus fixture now has an .osi.golden.yaml (IR -> OSI) and a .pull.golden.yaml (KC entries -> IR -> OSI). A reviewer sees the whole input and output as files and can diff the two to see exactly what a Knowledge Catalog round trip drops. Test files renamed to match their modules (osi_converter/kc_converter/pull_kc). - Document `kcmd pull` in docs/semantic-model.md: the --dry-run/--model flags, multiple models per entry group, last-write-wins overwrite policy, and the catalog-not-a-full-copy round-trip loss. --- toolbox/mdcode/docs/semantic-model.md | 65 ++++- .../semantic/deploy_knowledge_catalog.ts | 199 +------------ .../mdcode/src/libts/semantic/kc_converter.ts | 267 ++++++++++++++++++ .../src/libts/semantic/knowledge_catalog.ts | 263 +---------------- .../{serialize.ts => osi_converter.ts} | 19 +- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 182 ++++++++++++ toolbox/mdcode/src/tool/commands.ts | 5 +- .../sales_bq_graph_target.osi.golden.yaml | 29 ++ .../sales_bq_graph_target.pull.golden.yaml | 25 ++ .../star_orders_customer.osi.golden.yaml | 86 ++++++ .../star_orders_customer.pull.golden.yaml | 61 ++++ .../fixtures/tpcds_date_edge.osi.golden.yaml | 218 ++++++++++++++ .../fixtures/tpcds_date_edge.pull.golden.yaml | 147 ++++++++++ ...alog.read.test.ts => kc_converter.test.ts} | 63 ++++- ...erialize.test.ts => osi_converter.test.ts} | 52 +++- ...e_catalog.pull.test.ts => pull_kc.test.ts} | 6 +- 16 files changed, 1214 insertions(+), 473 deletions(-) create mode 100644 toolbox/mdcode/src/libts/semantic/kc_converter.ts rename toolbox/mdcode/src/libts/semantic/{serialize.ts => osi_converter.ts} (93%) create mode 100644 toolbox/mdcode/src/libts/semantic/pull_kc.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml rename toolbox/mdcode/tests/libts/semantic/{knowledge_catalog.read.test.ts => kc_converter.test.ts} (73%) rename toolbox/mdcode/tests/libts/semantic/{serialize.test.ts => osi_converter.test.ts} (81%) rename toolbox/mdcode/tests/libts/semantic/{deploy_knowledge_catalog.pull.test.ts => pull_kc.test.ts} (98%) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 19d192ba..7d3c28a2 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -13,8 +13,9 @@ model to two destinations at once: Both are generated from the same source document — you never author them separately, and a single `push` keeps them in sync. -This guide covers authoring, deploying, and updating a model. For the Ossie -document format itself, see [ossie.apache.org](https://ossie.apache.org/). +This guide covers authoring, deploying, pulling back, and updating a model. +For the Ossie document format itself, see +[ossie.apache.org](https://ossie.apache.org/). ## Prerequisites @@ -220,3 +221,63 @@ Every push prints one line per destination summarizing what it did. For a Deployed 1 BigQuery Graph(s). Wrote 5 new and 2 updated Knowledge Catalog entries; removed 1 orphaned entry; linked 2 relationships; unlinked 1 orphaned link. ``` + +## Pull + +`kcmd pull` is the inverse of push's Knowledge Catalog leg: it reads the +`semantic-*` entries back from the catalog and reconstructs local model +documents at `catalog/EntryGroups//.yaml`. Use it to +recover a workspace from a catalog someone else deployed, or to see what the +catalog actually holds. + +```bash +kcmd pull +``` + +Pull reads only from Knowledge Catalog (never BigQuery). Its coordinates come +from the same scope you authored under (`..`). + +| Flag | Effect | +|------|--------| +| `--dry-run` | Reconstruct from the catalog and report what would be written, but write no files. | +| `--model ` | Pull a single model by name; other models in the entry group are left alone. | + +One entry group can hold **many models** — each `semantic-model` entry is a +separate anchor, and pull reconstructs one document per anchor. `--model` +narrows both the fetch and the write to a single anchor. + +Pull writes with the same last-write-wins policy as the core pull: a model that +already exists locally is overwritten in place, and a local-only document (one +with no matching catalog entry) is left untouched — pull never deletes. + +> **Note — pull recovers the catalog, not your authored document.** The catalog +> stores only what push wrote to it (see the note under [What gets created in +> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)), so a pulled +> document comes back without the content the catalog never held: entity keys, +> `ai_context`, field labels, the original vendor SQL (`importedExpression`), +> and relationships (the graph edges live in the BigQuery property graph, not +> the catalog). A field's *role* survives as a bare `dimension: {}` marker, but +> its detail (`is_time`, and so on) does not. Keep your authored document as the +> source of truth; treat a pulled document as a faithful copy of the catalog +> metadata, not of the original model. + +## Permissions + +`push` needs access to whichever destinations you deploy to. + +**BigQuery** — for `--target bq` or `all`, and for the validation pre-flight: + +* `bigquery.jobs.create` in the deployment-target project — to run the deploy's + `CREATE OR REPLACE PROPERTY GRAPH` and the validation dry-run query +* read access on each entity's source table, so the dry-run can resolve it +* `bigquery.datasets.get` on the target dataset (region detection; optional — + push degrades gracefully without it) + +**Knowledge Catalog / Dataplex** — for `--target kc` or `all`: + +* `dataplex.entryGroups.useSemanticModelAspect` on the destination entry group +* `dataplex.entryGroups.useSchemaJoinEntryLink` and + `dataplex.entryGroups.useSchemaJoinAspect` when the model has relationships + +`kcmd pull` needs read access to the same entry group instead — to list its +entries and fetch each `semantic-*` entry with its aspects. diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 0bf69527..5676ec86 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -38,8 +38,7 @@ import {ApiResult} from '../gcp/api'; import * as context from '../gcp/context'; import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; -import {SemanticModel} from './ir'; -import {generateCatalogResources, idOf, KcResources, modelsFromCatalogResources} from './knowledge_catalog'; +import {generateCatalogResources, KcResources} from './knowledge_catalog'; import {LoadedModel} from './loader'; @@ -121,10 +120,6 @@ interface Counts { type EmittedModel = {model: string; resources: KcResources}; -// Upper bound on concurrent entry / entry-link writes within one model's wave -// (mirrors HYDRATE_CONCURRENCY on the pull side). -const WRITE_CONCURRENCY = 8; - // entries.create propagation retry: a just-created entry group can briefly 404. const ENTRY_CREATE_TRIES = 3; const ENTRY_CREATE_RETRY_MS = 3000; @@ -571,8 +566,7 @@ async function createEntries( const isMetric = (e: Entry) => (e.entryType ?? '').endsWith('/semantic-metric'); for (const wave of [children.filter(e => !isMetric(e)), children.filter(isMetric)]) { - const res = await mapConcurrent( - wave, WRITE_CONCURRENCY, e => writeEntry(cat, opts, e)); + const res = await Promise.all(wave.map(e => writeEntry(cat, opts, e))); const firstErr = res.find(r => r.error); if (firstErr) return {created, updated, error: firstErr.error}; for (const r of res) { @@ -599,8 +593,7 @@ async function createEntryLinks( cat: CatalogClient, opts: KcDeployOptions, links: EntryLink[]): Promise { if (!links.length) return {linked: 0}; - const res = await mapConcurrent( - links, WRITE_CONCURRENCY, l => writeEntryLink(cat, opts, l)); + const res = await Promise.all(links.map(l => writeEntryLink(cat, opts, l))); const firstErr = res.find(r => r.error); if (firstErr) return {linked: 0, error: firstErr.error}; return {linked: links.length}; @@ -693,6 +686,11 @@ function planSummary( } +// The id segment of a full entry/entryType resource name (after the last '/'). +function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + function isOk(res: {status: number}): boolean { return res.status === 200; } @@ -717,184 +715,3 @@ function isPropagating(res: {message?: string}): boolean { function errText(res: {status: number; message?: string}): string { return res.message?.trim() || `HTTP ${res.status}`; } - - -// --------------------------------------------------------------------------- -// Pull: Knowledge Catalog -> Semantic Model IR. -// -// The read counterpart of deployKnowledgeCatalog and the inverse of push. -// Unlike a write, a pull needs no server-side type provisioning -- only that -// the `semantic-*` entries exist. It enumerates the entry group, keeps the -// semantic entries, hydrates each one's aspect data (a BASIC list omits aspect -// data, so each entry is re-fetched with its aspect types -- an entity needs -// BOTH its `semantic-entity` aspect and the built-in `schema` aspect), and -// hands the hydrated entries to the pure reader (modelsFromCatalogResources). -// --------------------------------------------------------------------------- - -export interface KcPullOptions { - project: string; - location: string; - entryGroup: string; - model?: string; // limit to a single model by name (default: all) -} - -export interface KcPullResult { - models: SemanticModel[]; - warnings: string[]; -} - -// Upper bound on in-flight aspect-hydration fetches during a pull. -const HYDRATE_CONCURRENCY = 8; - -// Reads the semantic models back from a Knowledge Catalog entry group. Emits no -// console output; warnings (skipped entries, no match for --model, reader -// warnings) are returned for the caller to print. -export async function pullKnowledgeCatalog( - cat: CatalogClient, opts: KcPullOptions): Promise { - const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; - const warnings: string[] = []; - - // Enumerate the group (paging is inherently sequential) and pick the semantic - // entries, then hydrate their aspects concurrently: a BASIC list omits aspect - // data, so each entry needs its own lookupEntry, and those fetches are - // independent. The pool preserves input order so warnings stay deterministic. - const targets: {entry: Entry; aspectTypes: string[]}[] = []; - for await (const entry of cat.listEntries( - opts.project, opts.location, opts.entryGroup)) { - const aspectTypes = semanticAspectTypes(entry.entryType); - if (aspectTypes) targets.push({entry, aspectTypes}); - // else: not part of a semantic model; ignore it. - } - - // When scoped to one model, hydrate only that model's entries -- its anchor - // (matched by name) plus the children pointing at it. A list already carries - // entrySource + parentEntry, so this avoids fetching every other model's - // aspects. No match short-circuits with just the not-found warning. - let scoped = targets; - if (opts.model) { - scoped = scopeToModel(targets, opts.model); - if (!scoped.length) { - return { - models: [], - warnings: [ - `no semantic model named '${opts.model}' found in ${destination}` - ], - }; - } - } - - const fetched = await mapConcurrent( - scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { - const res = await cat.lookupEntry( - opts.project, opts.location, entry.name, aspectTypes); - if (res.status !== 200 || !res.result) { - return { - warning: `failed to fetch entry '${entry.name}' (status ${ - res.status}); skipped` - }; - } - return {entry: res.result}; - }); - - const hydrated: Entry[] = []; - for (const r of fetched) { - if (r.entry) - hydrated.push(r.entry); - else if (r.warning) - warnings.push(r.warning); - } - - const read = modelsFromCatalogResources(hydrated); - warnings.push(...read.warnings); - - // Defense in depth: keep only the requested model even if the reader surfaced - // another anchor (e.g. a child whose parentEntry pointed outside the scope). - let models = read.models; - if (opts.model) { - models = models.filter(m => m.name === opts.model); - if (!models.length) { - warnings.push( - `no semantic model named '${opts.model}' found in ${destination}`); - } - } - - return {models, warnings: [...new Set(warnings)]}; -} - - -// The aspect type resource names to hydrate for a semantic entry, derived from -// its entryType (the aspect types are the parallel resources in the same -// project/location). An entity carries two aspects: its `semantic-entity` -// aspect and the built-in `schema` aspect that holds its fields. Returns -// undefined for entries that are not part of a semantic model. -function semanticAspectTypes(entryType: string): string[]|undefined { - const marker = '/entryTypes/'; - const idx = entryType?.indexOf(marker) ?? -1; - if (idx < 0) return undefined; - const typeBase = entryType.slice(0, idx); - const t = entryType.slice(idx + marker.length); - const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; - switch (t) { - case 'semantic-model': - return [aspectType('semantic-model')]; - case 'semantic-entity': - return [aspectType('semantic-entity'), aspectType('schema')]; - case 'semantic-metric': - return [aspectType('semantic-metric')]; - default: - return undefined; - } -} - - -// Restricts hydration targets to a single model: the semantic-model anchor -// whose name (entrySource.displayName, else the entry id) matches `model`, plus -// every child entry whose parentEntry is that anchor. Uses only list-level -// fields (no aspect data), so it runs before hydration and avoids fetching -// unrelated models' aspects. Returns [] when no anchor matches. -function scopeToModel( - targets: {entry: Entry; aspectTypes: string[]}[], - model: string): {entry: Entry; aspectTypes: string[]}[] { - const isAnchor = (t: {entry: Entry}) => - !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); - const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); - const matchedAnchorNames = new Set( - targets.filter(isAnchor) - .filter( - t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === - model) - .map(t => t.entry.name)); - if (!matchedAnchorNames.size) return []; - // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds - // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a - // project-id normalization mismatch) still belongs to it. Without this a - // scoped pull would drop children a full pull keeps. - const soleAnchor = - allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; - const soleMatched = - soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); - return targets.filter( - t => matchedAnchorNames.has(t.entry.name) || - matchedAnchorNames.has(t.entry.parentEntry ?? '') || - (soleMatched && !isAnchor(t) && - !allAnchorNames.has(t.entry.parentEntry ?? ''))); -} - - -// Maps `items` through `fn` with at most `limit` calls in flight, returning -// results in input order (so downstream ordering stays deterministic). -async function mapConcurrent( - items: T[], limit: number, fn: (item: T) => Promise): Promise { - const results: R[] = new Array(items.length); - let next = 0; - async function worker(): Promise { - while (next < items.length) { - const i = next++; - results[i] = await fn(items[i]); - } - } - const workers = - Array.from({length: Math.min(limit, items.length)}, () => worker()); - await Promise.all(workers); - return results; -} diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts new file mode 100644 index 00000000..233f7b5e --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -0,0 +1,267 @@ +// Knowledge Catalog <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// READ direction -- Knowledge Catalog entries -> IR (`modelsFromCatalogResources` +// and its helpers). The WRITE direction (IR -> KC entries, +// `generateCatalogResources`) still lives in `knowledge_catalog.ts` and moves +// here once PR4 (the KC push line) merges, at which point this becomes the full +// two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is +// the KC emit code?" -> `knowledge_catalog.ts`. +// +// The reader is the inverse of `generateCatalogResources`: it reconstructs the +// IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). +// `semantic-entity` / `semantic-metric` entries are grouped under their +// `semantic-model` anchor via `parentEntry`; entries of other types are ignored. +// Resources are matched by type-name SUFFIX, so a reader need not know which +// system-type project/location the emitter used. +// +// Fidelity is bounded by what the emitter persisted, so this read is the inverse +// of the WRITE, not of the authored document. It recovers names, descriptions, +// data sources, field datatypes (via the schema aspect) and DIMENSION roles, +// field/metric expressions, and each metric's attach entity (re-derived from its +// expression, as the loader does). It cannot recover what the emitter does not +// write: entity keys/unique keys, `ai_context`, field labels, `importedDialect`, +// `custom_extensions`, and relationships (the graph edges live in the BigQuery +// property graph, not the catalog). + +import type {Entry} from '../gcp/dataplex'; +import {DataType, Entity, Field, Metric, SemanticModel} from './ir'; +import {referencedEntityNames} from './sql_expr_utils'; + +export interface ReadResult { + models: SemanticModel[]; + warnings: string[]; +} + +/** + * Reconstructs the Semantic Model IR from Knowledge Catalog entries. + * + * Returns one model per `semantic-model` anchor plus any warnings (no anchor, + * an orphaned child, an entry missing its aspect data). Entries must already be + * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a + * BASIC list omits aspect data, so the puller re-fetches each entry first. + */ +export function modelsFromCatalogResources(entries: Entry[]): ReadResult { + const warnings: string[] = []; + + const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); + const entityEntries = + entries.filter(e => semanticType(e) === 'semantic-entity'); + const metricEntries = + entries.filter(e => semanticType(e) === 'semantic-metric'); + + if (!anchors.length) { + warnings.push('no semantic-model entry found; nothing to reconstruct'); + return {models: [], warnings: [...new Set(warnings)]}; + } + + // A child belongs to its anchor by parentEntry. When there is exactly one + // anchor, children whose parentEntry does not resolve (e.g. a project-id + // normalization mismatch) are still attached to it rather than dropped. + const anchorNames = new Set(anchors.map(a => a.name)); + const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; + const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( + e => e.parentEntry === anchorName || + (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); + + const models = anchors.map(anchor => { + const name = anchor.entrySource?.displayName ?? idOf(anchor.name); + + const entities = childrenOf(anchor.name, entityEntries) + .map(e => readEntity(e, warnings)); + const entityNames = entities.map(e => e.name); + const metrics = childrenOf(anchor.name, metricEntries) + .map(e => readMetric(e, entityNames, warnings)); + + // Relationships are not published to the catalog (see the file header), so + // a reconstructed model always has an empty edge set. + const model: SemanticModel = {name, entities, relationships: [], metrics}; + const description = anchor.entrySource?.description; + if (description !== undefined) model.description = description; + return model; + }); + + // Flag children that resolved to no anchor at all (only possible with + // multiple anchors, where the sole-anchor fallback does not apply). + if (!soleAnchor) { + for (const child of [...entityEntries, ...metricEntries]) { + if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { + warnings.push(`entry '${ + child.name}' has no resolvable parent semantic-model; omitted`); + } + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// Reconstructs an entity from its `semantic-entity` aspect (the backing source) +// and the built-in `schema` aspect (its fields). Keys are not persisted by the +// emitter and so come back empty. +function readEntity(entry: Entry, warnings: string[]): Entity { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const semantic = aspectData(entry, 'semantic-entity'); + const schema = aspectData(entry, 'schema'); + if (!Object.keys(semantic).length) { + warnings.push(`entity '${ + name}': no semantic-entity aspect data (fetch with the aspect type)`); + } + + const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); + if (!dataSource) { + warnings.push( + `entity '${name}': no backing data source in the semantic-entity ` + + `aspect; 'source' will be empty and the entity may not load`); + } + const entity: Entity = { + name, + dataSource, + keys: [], // not persisted by the emitter; unrecoverable on read + fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + }; + const description = entry.entrySource?.description; + if (description !== undefined) entity.description = description; + return entity; +} + + +// Reconstructs a field from one `schema` aspect field record, inverting +// schemaAspectData: the datatype from dataType/metadataType, expressions from +// the nested `semantics` block, and the DIMENSION role back to a dimension +// marker. +function readField(fd: any, entityName: string, warnings: string[]): Field { + const name = fd?.name; + if (name === undefined || name === '') { + warnings.push( + `entity '${entityName}': a schema field is missing its name; the ` + + `field may not load`); + } + const field: Field = {name}; + const sem = fd?.semantics ?? {}; + if (sem.expression !== undefined) field.expression = sem.expression; + const type = irDataType(fd?.dataType, fd?.metadataType); + if (type !== undefined) field.type = type; + if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.description !== undefined) field.description = fd.description; + return field; +} + + +// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` +// is re-derived from the expression (as the loader does) rather than read from +// the aspect, so it stays consistent with the reconstructed entity set. +function readMetric( + entry: Entry, entityNames: string[], warnings: string[]): Metric { + const name = entry.entrySource?.displayName ?? idOf(entry.name); + const data = aspectData(entry, 'semantic-metric'); + if (data.expression === undefined) { + warnings.push(`metric '${name}': no expression in semantic-metric aspect`); + } + + const metric: Metric = {name}; + if (data.expression !== undefined) metric.expression = data.expression; + const exprForRefs = data.expression ?? ''; + const referenced = referencedEntityNames(exprForRefs, entityNames); + if (referenced.length === 1) { + metric.entity = referenced[0]; + } else if (exprForRefs && !referenced.length) { + // Parity with the loader's convertMetric: an expression that qualifies no + // known entity is flagged as potentially unplaceable downstream. + warnings.push( + `metric '${name}': expression references no known entity; it may not ` + + `be placeable downstream`); + } + // The emitter writes a required dataType, defaulting a typeless metric to + // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric + // authored without a datatype round-trips as an explicit Decimal rather than + // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) + const type = irDataType(data.dataType, undefined); + if (type !== undefined) metric.type = type; + const description = entry.entrySource?.description; + if (description !== undefined) metric.description = description; + return metric; +} + + +// The inverse of columnDataType/columnMetadataType: maps the schema aspect's +// dataType (disambiguated by metadataType only for the STRING family) back to +// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read +// as un-typed (undefined) -- the loader's default -- since the emitter cannot +// distinguish an authored `String` from an un-typed field (both emit STRING). +function irDataType(dataType: string|undefined, metadataType: string|undefined): + DataType|undefined { + switch (dataType) { + case 'INT64': + return 'Integer'; + case 'NUMERIC': + return 'Decimal'; + case 'FLOAT64': + return 'Float'; + case 'BOOL': + return 'Boolean'; + case 'DATE': + return 'Date'; + case 'TIME': + return 'Time'; + case 'DATETIME': + return 'DateTime'; + case 'TIMESTAMP': + return 'DateTimeTz'; + case 'STRING': + return metadataType === 'OTHER' ? 'Opaque' : undefined; + default: + return undefined; + } +} + + +// The inverse of resourcePath: a BigQuery linked-resource URI becomes the +// canonical `project.dataset.table` string; anything else (a verbatim query or +// a passthrough reference) is returned unchanged. +function dataSourceFromResource(resource: string|undefined): string { + const value = (resource ?? '').trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + + +// The bare `semantic-*` type of an entry, matched by entryType suffix so the +// system-type project/location need not be known. Returns undefined for entries +// that are not part of a semantic model. +function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| + 'semantic-metric'|undefined { + for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as + const) { + if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; + } + return undefined; +} + + +// The `data` payload of an entry's aspect of the given bare type, matched by +// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` +// suffix (robust to whichever system-type project/location the emitter used). +// Returns an empty object when the aspect is absent. +function aspectData(entry: Entry, type: string): Record { + const aspects = entry.aspects ?? {}; + for (const [key, aspect] of Object.entries(aspects)) { + if (key.endsWith(`.${type}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + + +// The id segment of a full entry resource name (after the last '/'). +export function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index 5e5860c9..a3051b22 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -45,8 +45,7 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {bigQueryGraphTargets} from './deploy_bigquery'; -import {DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; -import {referencedEntityNames} from './sql_expr_utils'; +import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -540,263 +539,3 @@ function linkSlug(s: string): string { function unquote(part: string): string { return part.replace(/^[`"]/, '').replace(/[`"]$/, ''); } - - -// --------------------------------------------------------------------------- -// Reader: Knowledge Catalog entries -> Semantic Model IR. -// -// The inverse of generateCatalogResources: it reconstructs the IR from the -// entries a pull hydrated (see deploy_knowledge_catalog.pullKnowledgeCatalog). -// `semantic-entity` / `semantic-metric` entries are grouped under their -// `semantic-model` anchor via `parentEntry`; entries of other types are -// ignored. Resources are matched by type-name SUFFIX, so a reader need not know -// which system-type project/location the emitter used. -// -// Fidelity is bounded by what the emitter persisted, so this read is the -// inverse of the WRITE, not of the authored document. It recovers names, -// descriptions, data sources, field datatypes (via the schema aspect) and -// DIMENSION roles, field/metric expressions, and each -// metric's attach entity (re-derived from its expression, as the loader does). -// It cannot recover what the emitter does not write: entity keys/unique keys, -// `ai_context`, field labels, `importedDialect`, `custom_extensions`, and -// relationships (the graph edges live in the BigQuery property graph, not the -// catalog). -// --------------------------------------------------------------------------- - -export interface ReadResult { - models: SemanticModel[]; - warnings: string[]; -} - -/** - * Reconstructs the Semantic Model IR from Knowledge Catalog entries. - * - * Returns one model per `semantic-model` anchor plus any warnings (no anchor, - * an orphaned child, an entry missing its aspect data). Entries must already be - * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a - * BASIC list omits aspect data, so the puller re-fetches each entry first. - */ -export function modelsFromCatalogResources(entries: Entry[]): ReadResult { - const warnings: string[] = []; - - const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); - const entityEntries = - entries.filter(e => semanticType(e) === 'semantic-entity'); - const metricEntries = - entries.filter(e => semanticType(e) === 'semantic-metric'); - - if (!anchors.length) { - warnings.push('no semantic-model entry found; nothing to reconstruct'); - return {models: [], warnings: [...new Set(warnings)]}; - } - - // A child belongs to its anchor by parentEntry. When there is exactly one - // anchor, children whose parentEntry does not resolve (e.g. a project-id - // normalization mismatch) are still attached to it rather than dropped. - const anchorNames = new Set(anchors.map(a => a.name)); - const soleAnchor = anchors.length === 1 ? anchors[0].name : undefined; - const childrenOf = (anchorName: string, pool: Entry[]) => pool.filter( - e => e.parentEntry === anchorName || - (soleAnchor === anchorName && !anchorNames.has(e.parentEntry ?? ''))); - - const models = anchors.map(anchor => { - const name = anchor.entrySource?.displayName ?? idOf(anchor.name); - - const entities = childrenOf(anchor.name, entityEntries) - .map(e => readEntity(e, warnings)); - const entityNames = entities.map(e => e.name); - const metrics = childrenOf(anchor.name, metricEntries) - .map(e => readMetric(e, entityNames, warnings)); - - // Relationships are not published to the catalog (see the file header), so - // a reconstructed model always has an empty edge set. - const model: SemanticModel = {name, entities, relationships: [], metrics}; - const description = anchor.entrySource?.description; - if (description !== undefined) model.description = description; - return model; - }); - - // Flag children that resolved to no anchor at all (only possible with - // multiple anchors, where the sole-anchor fallback does not apply). - if (!soleAnchor) { - for (const child of [...entityEntries, ...metricEntries]) { - if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { - warnings.push(`entry '${ - child.name}' has no resolvable parent semantic-model; omitted`); - } - } - } - - return {models, warnings: [...new Set(warnings)]}; -} - - -// Reconstructs an entity from its `semantic-entity` aspect (the backing source) -// and the built-in `schema` aspect (its fields). Keys are not persisted by the -// emitter and so come back empty. -function readEntity(entry: Entry, warnings: string[]): Entity { - const name = entry.entrySource?.displayName ?? idOf(entry.name); - const semantic = aspectData(entry, 'semantic-entity'); - const schema = aspectData(entry, 'schema'); - if (!Object.keys(semantic).length) { - warnings.push(`entity '${ - name}': no semantic-entity aspect data (fetch with the aspect type)`); - } - - const dataSource = dataSourceFromResource(semantic?.source?.resources?.[0]); - if (!dataSource) { - warnings.push( - `entity '${name}': no backing data source in the semantic-entity ` + - `aspect; 'source' will be empty and the entity may not load`); - } - const entity: Entity = { - name, - dataSource, - keys: [], // not persisted by the emitter; unrecoverable on read - fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), - }; - const description = entry.entrySource?.description; - if (description !== undefined) entity.description = description; - return entity; -} - - -// Reconstructs a field from one `schema` aspect field record, inverting -// schemaAspectData: the datatype from dataType/metadataType, expressions from -// the nested `semantics` block, and the DIMENSION role back to a dimension -// marker. -function readField(fd: any, entityName: string, warnings: string[]): Field { - const name = fd?.name; - if (name === undefined || name === '') { - warnings.push( - `entity '${entityName}': a schema field is missing its name; the ` + - `field may not load`); - } - const field: Field = {name}; - const sem = fd?.semantics ?? {}; - if (sem.expression !== undefined) field.expression = sem.expression; - const type = irDataType(fd?.dataType, fd?.metadataType); - if (type !== undefined) field.type = type; - if (sem.role === 'DIMENSION') field.dimension = {}; - if (fd?.description !== undefined) field.description = fd.description; - return field; -} - - -// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` -// is re-derived from the expression (as the loader does) rather than read from -// the aspect, so it stays consistent with the reconstructed entity set. -function readMetric( - entry: Entry, entityNames: string[], warnings: string[]): Metric { - const name = entry.entrySource?.displayName ?? idOf(entry.name); - const data = aspectData(entry, 'semantic-metric'); - if (data.expression === undefined) { - warnings.push(`metric '${name}': no expression in semantic-metric aspect`); - } - - const metric: Metric = {name}; - if (data.expression !== undefined) metric.expression = data.expression; - const exprForRefs = data.expression ?? ''; - const referenced = referencedEntityNames(exprForRefs, entityNames); - if (referenced.length === 1) { - metric.entity = referenced[0]; - } else if (exprForRefs && !referenced.length) { - // Parity with the loader's convertMetric: an expression that qualifies no - // known entity is flagged as potentially unplaceable downstream. - warnings.push( - `metric '${name}': expression references no known entity; it may not ` + - `be placeable downstream`); - } - // The emitter writes a required dataType, defaulting a typeless metric to - // NUMERIC (see metricAspectData); NUMERIC maps back to Decimal, so a metric - // authored without a datatype round-trips as an explicit Decimal rather than - // un-typed. (Dimensions differ: their STRING default reads back as un-typed.) - const type = irDataType(data.dataType, undefined); - if (type !== undefined) metric.type = type; - const description = entry.entrySource?.description; - if (description !== undefined) metric.description = description; - return metric; -} - - -// The inverse of columnDataType/columnMetadataType: maps the schema aspect's -// dataType (disambiguated by metadataType only for the STRING family) back to -// the IR's logical DataType. STRING + OTHER is Opaque; a plain STRING is read -// as un-typed (undefined) -- the loader's default -- since the emitter cannot -// distinguish an authored `String` from an un-typed field (both emit STRING). -function irDataType(dataType: string|undefined, metadataType: string|undefined): - DataType|undefined { - switch (dataType) { - case 'INT64': - return 'Integer'; - case 'NUMERIC': - return 'Decimal'; - case 'FLOAT64': - return 'Float'; - case 'BOOL': - return 'Boolean'; - case 'DATE': - return 'Date'; - case 'TIME': - return 'Time'; - case 'DATETIME': - return 'DateTime'; - case 'TIMESTAMP': - return 'DateTimeTz'; - case 'STRING': - return metadataType === 'OTHER' ? 'Opaque' : undefined; - default: - return undefined; - } -} - - -// The inverse of resourcePath: a BigQuery linked-resource URI becomes the -// canonical `project.dataset.table` string; anything else (a verbatim query or -// a passthrough reference) is returned unchanged. -function dataSourceFromResource(resource: string|undefined): string { - const value = (resource ?? '').trim(); - const m = value.match( - /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); - return m ? `${m[1]}.${m[2]}.${m[3]}` : value; -} - - -// The bare `semantic-*` type of an entry, matched by entryType suffix so the -// system-type project/location need not be known. Returns undefined for entries -// that are not part of a semantic model. -function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| - 'semantic-metric'|undefined { - for (const t of ['semantic-model', 'semantic-entity', 'semantic-metric'] as - const) { - if (entry.entryType?.endsWith(`/entryTypes/${t}`)) return t; - } - return undefined; -} - - -// The `data` payload of an entry's aspect of the given bare type, matched by -// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` -// suffix (robust to whichever system-type project/location the emitter used). -// Returns an empty object when the aspect is absent. -function aspectData(entry: Entry, type: string): Record { - const aspects = entry.aspects ?? {}; - for (const [key, aspect] of Object.entries(aspects)) { - if (key.endsWith(`.${type}`) || - aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { - return aspect.data ?? {}; - } - } - return {}; -} - - -// The id segment of a full entry resource name (after the last '/'). -export function idOf(name: string): string { - return name.split('/').pop() ?? name; -} - - -function asArray(value: any): any[] { - return Array.isArray(value) ? value : []; -} diff --git a/toolbox/mdcode/src/libts/semantic/serialize.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts similarity index 93% rename from toolbox/mdcode/src/libts/semantic/serialize.ts rename to toolbox/mdcode/src/libts/semantic/osi_converter.ts index dd215100..61c4a99b 100644 --- a/toolbox/mdcode/src/libts/semantic/serialize.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -1,9 +1,16 @@ -// Serializes the Semantic Model IR (./ir) back to the open AI-first semantics -// format (YAML). This is the inverse of `loader.ts`: `loader` reads authored -// YAML into the IR; this module writes the IR out as a YAML document the loader -// can read back. It is the local-workspace sink for `pull` (Knowledge Catalog -// -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR -> a -// destination. +// OSI (open, AI-first semantics) <-> Semantic Model IR converter. +// +// SCAFFOLD (naming for the end state): this file currently holds only the +// WRITE direction -- IR -> OSI YAML (`serializeModel`). The READ direction +// (OSI YAML -> IR) still lives in `loader.ts` and moves here once PR4 (the +// KC push line) merges, at which point this becomes the full two-way +// converter. Until then, "where is the OSI parser?" -> `loader.ts`. +// +// Serializing is the inverse of `loader.ts`: `loader` reads authored YAML +// into the IR; this module writes the IR out as a YAML document the loader +// can read back. It is the local-workspace sink for `pull` (Knowledge +// Catalog -> IR -> YAML), the counterpart to how `push` compiles YAML -> IR +// -> a destination. // // Fidelity is at the IR level, not byte-for-byte with a hand-authored file. The // loader normalizes several authoring conveniences into the IR at load time, so diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts new file mode 100644 index 00000000..f03edc3f --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -0,0 +1,182 @@ +// Pull: fetch a semantic model from a live Knowledge Catalog into the IR. +// +// The read-direction counterpart of `deployKnowledgeCatalog` (push, in +// `deploy_knowledge_catalog.ts`). Unlike a write, a pull needs no server-side +// type provisioning -- only that the `semantic-*` entries exist. It enumerates +// the entry group, keeps the semantic entries, hydrates each one's aspect data +// (a BASIC list omits aspect data, so each entry is re-fetched with its aspect +// types -- an entity needs BOTH its `semantic-entity` aspect and the built-in +// `schema` aspect), and hands the hydrated entries to the pure reader +// (`kc_converter.modelsFromCatalogResources`). + +import {CatalogClient, Entry} from '../gcp/dataplex'; +import {SemanticModel} from './ir'; +import {idOf, modelsFromCatalogResources} from './kc_converter'; + +export interface KcPullOptions { + project: string; + location: string; + entryGroup: string; + model?: string; // limit to a single model by name (default: all) +} + +export interface KcPullResult { + models: SemanticModel[]; + warnings: string[]; +} + +// Upper bound on in-flight aspect-hydration fetches during a pull. +const HYDRATE_CONCURRENCY = 8; + +// Reads the semantic models back from a Knowledge Catalog entry group. Emits no +// console output; warnings (skipped entries, no match for --model, reader +// warnings) are returned for the caller to print. +export async function pullKnowledgeCatalog( + cat: CatalogClient, opts: KcPullOptions): Promise { + const destination = `${opts.project}.${opts.location}.${opts.entryGroup}`; + const warnings: string[] = []; + + // Enumerate the group (paging is inherently sequential) and pick the semantic + // entries, then hydrate their aspects concurrently: a BASIC list omits aspect + // data, so each entry needs its own lookupEntry, and those fetches are + // independent. The pool preserves input order so warnings stay deterministic. + const targets: {entry: Entry; aspectTypes: string[]}[] = []; + for await (const entry of cat.listEntries( + opts.project, opts.location, opts.entryGroup)) { + const aspectTypes = semanticAspectTypes(entry.entryType); + if (aspectTypes) targets.push({entry, aspectTypes}); + // else: not part of a semantic model; ignore it. + } + + // When scoped to one model, hydrate only that model's entries -- its anchor + // (matched by name) plus the children pointing at it. A list already carries + // entrySource + parentEntry, so this avoids fetching every other model's + // aspects. No match short-circuits with just the not-found warning. + let scoped = targets; + if (opts.model) { + scoped = scopeToModel(targets, opts.model); + if (!scoped.length) { + return { + models: [], + warnings: [ + `no semantic model named '${opts.model}' found in ${destination}` + ], + }; + } + } + + const fetched = await mapConcurrent( + scoped, HYDRATE_CONCURRENCY, async ({entry, aspectTypes}) => { + const res = await cat.lookupEntry( + opts.project, opts.location, entry.name, aspectTypes); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry '${entry.name}' (status ${ + res.status}); skipped` + }; + } + return {entry: res.result}; + }); + + const hydrated: Entry[] = []; + for (const r of fetched) { + if (r.entry) + hydrated.push(r.entry); + else if (r.warning) + warnings.push(r.warning); + } + + const read = modelsFromCatalogResources(hydrated); + warnings.push(...read.warnings); + + // Defense in depth: keep only the requested model even if the reader surfaced + // another anchor (e.g. a child whose parentEntry pointed outside the scope). + let models = read.models; + if (opts.model) { + models = models.filter(m => m.name === opts.model); + if (!models.length) { + warnings.push( + `no semantic model named '${opts.model}' found in ${destination}`); + } + } + + return {models, warnings: [...new Set(warnings)]}; +} + + +// The aspect type resource names to hydrate for a semantic entry, derived from +// its entryType (the aspect types are the parallel resources in the same +// project/location). An entity carries two aspects: its `semantic-entity` +// aspect and the built-in `schema` aspect that holds its fields. Returns +// undefined for entries that are not part of a semantic model. +function semanticAspectTypes(entryType: string): string[]|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + const typeBase = entryType.slice(0, idx); + const t = entryType.slice(idx + marker.length); + const aspectType = (name: string) => `${typeBase}/aspectTypes/${name}`; + switch (t) { + case 'semantic-model': + return [aspectType('semantic-model')]; + case 'semantic-entity': + return [aspectType('semantic-entity'), aspectType('schema')]; + case 'semantic-metric': + return [aspectType('semantic-metric')]; + default: + return undefined; + } +} + + +// Restricts hydration targets to a single model: the semantic-model anchor +// whose name (entrySource.displayName, else the entry id) matches `model`, plus +// every child entry whose parentEntry is that anchor. Uses only list-level +// fields (no aspect data), so it runs before hydration and avoids fetching +// unrelated models' aspects. Returns [] when no anchor matches. +function scopeToModel( + targets: {entry: Entry; aspectTypes: string[]}[], + model: string): {entry: Entry; aspectTypes: string[]}[] { + const isAnchor = (t: {entry: Entry}) => + !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); + const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = new Set( + targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === + model) + .map(t => t.entry.name)); + if (!matchedAnchorNames.size) return []; + // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds + // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a + // project-id normalization mismatch) still belongs to it. Without this a + // scoped pull would drop children a full pull keeps. + const soleAnchor = + allAnchorNames.size === 1 ? [...allAnchorNames][0] : undefined; + const soleMatched = + soleAnchor !== undefined && matchedAnchorNames.has(soleAnchor); + return targets.filter( + t => matchedAnchorNames.has(t.entry.name) || + matchedAnchorNames.has(t.entry.parentEntry ?? '') || + (soleMatched && !isAnchor(t) && + !allAnchorNames.has(t.entry.parentEntry ?? ''))); +} + + +// Maps `items` through `fn` with at most `limit` calls in flight, returning +// results in input order (so downstream ordering stays deterministic). +async function mapConcurrent( + items: T[], limit: number, fn: (item: T) => Promise): Promise { + const results: R[] = new Array(items.length); + let next = 0; + async function worker(): Promise { + while (next < items.length) { + const i = next++; + results[i] = await fn(items[i]); + } + } + const workers = + Array.from({length: Math.min(limit, items.length)}, () => worker()); + await Promise.all(workers); + return results; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 59ce84e1..1c67d76f 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -14,7 +14,8 @@ import * as deploy from '../libts/semantic/deploy_bigquery'; import * as kc from '../libts/semantic/deploy_knowledge_catalog'; import {BigQueryClient} from '../libts/gcp/bigquery'; import {LoadedModel, loadSemanticModels} from '../libts/semantic/loader'; -import { serializeModel } from '../libts/semantic/serialize'; +import {pullKnowledgeCatalog} from '../libts/semantic/pull_kc'; +import {serializeModel} from '../libts/semantic/osi_converter'; import {validateBigQueryDataSources, validatePushRequirements} from '../libts/semantic/validate'; @@ -387,7 +388,7 @@ async function pullSemanticModel( : 'Pulling semantic model from Knowledge Catalog...'); const catalog = new dataplex.CatalogClient(ctx); - const result = await kc.pullKnowledgeCatalog(catalog, { + const result = await pullKnowledgeCatalog(catalog, { project: source.project, location: source.location, entryGroup: source.entryGroup, diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml new file mode 100644 index 00000000..ffdfa798 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.osi.golden.yaml @@ -0,0 +1,29 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' + datasets: + - name: orders + source: demo.sales.orders + primary_key: + - o_orderkey + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml new file mode 100644 index 00000000..f9e42580 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -0,0 +1,25 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + datasets: + - name: orders + source: demo.sales.orders + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml new file mode 100644 index 00000000..40c22466 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.osi.golden.yaml @@ -0,0 +1,86 @@ +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + ai_context: + instructions: Use this model for order analysis. + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets": + ["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' + datasets: + - name: orders + source: samples.tpch.orders + primary_key: + - o_orderkey + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + label: Order Date + dimension: + is_time: true + ai_context: + synonyms: + - order date + - date + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + - name: customer + source: samples.tpch.customer + primary_key: + - c_custkey + fields: + - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name + description: Customer name + relationships: + - name: orders_to_customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + description: Total order revenue + ai_context: + synonyms: + - revenue + - sales + - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml new file mode 100644 index 00000000..bf7cfc8d --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -0,0 +1,61 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: sales + description: Sales orders with customer attributes + datasets: + - name: orders + source: samples.tpch.orders + description: One row per order + fields: + - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey + description: Order identifier + - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey + - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + dimension: {} + - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice + - name: customer + source: samples.tpch.customer + fields: + - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey + - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name + description: Customer name + metrics: + - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) + datatype: Decimal + description: Total order revenue + - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) + datatype: Decimal + description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml new file mode 100644 index 00000000..27acf1e6 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.osi.golden.yaml @@ -0,0 +1,218 @@ +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + primary_key: + - ss_item_sk + - ss_ticket_number + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_item_sk}" + dimension: + is_time: false + - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_ticket_number}" + dimension: + is_time: false + - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_customer_sk}" + dimension: + is_time: false + - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + - dialect: MAQL + expression: "{label/store_sales.attr.store_sales.ss_store_sk}" + dimension: + is_time: false + - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_quantity}" + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_sales_price}" + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_ext_sales_price}" + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + - dialect: MAQL + expression: "{fact/store_sales.fact.store_sales.ss_net_profit}" + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + primary_key: + - c_customer_sk + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: + is_time: false + - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: + is_time: false + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: + is_time: false + description: Customer last name + - name: item + source: tpcds.public.item + primary_key: + - i_item_sk + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: + is_time: false + - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: + is_time: false + - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: + is_time: false + - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + primary_key: + - s_store_sk + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: + is_time: false + - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: + is_time: false + - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: + is_time: false + - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: + is_time: false + - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes + custom_extensions: + - vendor_name: GOODDATA + data: '{"date_dimension": true, "granularities": ["DAY", "WEEK", "MONTH", + "QUARTER", "YEAR"]}' + relationships: + - name: store_sales_to_date_dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store_sales_to_item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store_sales_to_store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml new file mode 100644 index 00000000..7437f8ac --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -0,0 +1,147 @@ +# (no warnings) +version: 0.2.0.dev0 +semantic_model: + - name: tpcds_model + description: TPC-DS retail model + datasets: + - name: store_sales + source: tpcds.public.store_sales + description: Fact table containing all store sales transactions + fields: + - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + dimension: {} + - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + dimension: {} + - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + dimension: {} + - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + dimension: {} + - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + description: Quantity of items sold + - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + description: Sales price per unit + - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + description: Net profit from the sale + - name: customer + source: tpcds.public.customer + description: Customer dimension with demographic information + fields: + - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: {} + - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: {} + description: Customer first name + - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: {} + description: Customer last name + - name: item + source: tpcds.public.item + description: Item/Product dimension + fields: + - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: {} + - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: {} + - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: {} + - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price + description: Current price of the item + - name: store + source: tpcds.public.store + description: Store dimension with location attributes + fields: + - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: {} + - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: {} + - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: {} + - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: {} + - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees + description: Number of employees at the store + - name: date_dim + source: sqlgen-testing.demo.date_dim + description: Date dimension with calendar attributes diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts similarity index 73% rename from toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts rename to toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index a8d5e447..d5ecb145 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.read.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -1,5 +1,5 @@ -// Behavior specification for the Knowledge Catalog reader -// (modelsFromCatalogResources in src/libts/semantic/knowledge_catalog.ts). +// Behavior specification for the KC converter's read direction +// (modelsFromCatalogResources in src/libts/semantic/kc_converter.ts). // // The reader is the inverse of the emitter (generateCatalogResources). The // central guarantee is an emitter -> reader round trip: emit a model's entries, @@ -12,9 +12,15 @@ // re-derivation, and parent/anchor grouping). import {describe, expect, test} from 'bun:test'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {generateCatalogResources, modelsFromCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {serializeModel} from '../../../src/libts/semantic/osi_converter'; + +const FIXTURES = path.join(__dirname, 'fixtures'); const OPTS = { project: 'dest', @@ -256,3 +262,54 @@ describe('metric expression referencing no known entity', () => { .toBe(true); }); }); + + +// -- Golden pull: the whole KC entries -> IR -> OSI YAML output. -- +// +// The round trip above proves the reader inverts the emitter in memory; this +// pins the reviewable artifact. For each corpus fixture it reads the committed +// emitter golden (`.knowledge_catalog.golden.json` -- the exact entries +// a push produced) back through the reader and serializes the reconstructed IR +// to `.pull.golden.yaml`. Open that next to the fixture's +// `.osi.golden.yaml` to see, as whole files, what a Knowledge Catalog round trip +// preserves and what it drops (keys, ai_context, labels, relationships). +// +// Regenerate after an intentional reader/serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts +describe('golden pull: each corpus KC golden reconstructs to its exact YAML', + () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + const kcGoldenPath = (fixture: string) => path.join( + FIXTURES, + fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + const pullGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const {models, warnings} = modelsFromCatalogResources(kc.entries); + // Reader warnings ride along as YAML comments so the golden shows + // the full outcome, not just the recovered document. + const header = warnings.length ? + warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : + '# (no warnings)\n'; + const actual = + header + models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = pullGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts similarity index 81% rename from toolbox/mdcode/tests/libts/semantic/serialize.test.ts rename to toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts index 81c4454c..82943898 100644 --- a/toolbox/mdcode/tests/libts/semantic/serialize.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts @@ -1,7 +1,7 @@ -// Behavior specification for the semantic-model serializer -// (src/libts/semantic/serialize.ts). +// Behavior specification for the OSI converter's serialize direction +// (serializeModel in src/libts/semantic/osi_converter.ts). // -// serialize.ts is the inverse of loader.ts: IR -> open-format YAML. The +// The serializer is the inverse of loader.ts: IR -> open-format YAML. The // strongest guarantee is a round trip through the loader -- load a fixture to // the IR, serialize it, load the serialized text again, and assert the two IRs // are identical. That pins IR-level fidelity across every feature the loader @@ -17,7 +17,7 @@ import * as yaml from 'yaml'; import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; import {loadModels} from '../../../src/libts/semantic/loader'; -import {modelDocument, serializeModel} from '../../../src/libts/semantic/serialize'; +import {modelDocument, serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -288,3 +288,47 @@ describe('serialize flags loader-invalid reconstructions', () => { expect(labels).toContain('BIGQUERY'); }); }); + + +// -- Golden corpus: the whole IR -> OSI YAML output, reviewable as a file. -- +// +// The round-trip tests above prove IR-level stability but never pin the exact +// YAML text. These goldens capture the full serialized document for a small +// corpus so a reviewer can open a `.yaml` next to its `.osi.golden.yaml` and see +// exactly what the serializer emits. The same corpus backs kc_converter's +// `.pull.golden.yaml`, so diffing the two shows what a Knowledge Catalog round +// trip loses relative to the authored source. +// +// Regenerate after an intentional serializer change: +// UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/osi_converter.test.ts +describe('golden OSI document: each corpus fixture serializes to its exact YAML', + () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the KC e2e/pull goldens, so the OSI golden + // and the pull golden are directly comparable. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + const osiGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.osi.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + const models = loadModels(text, LOAD).models; + const actual = models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = osiGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts similarity index 98% rename from toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts rename to toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index e31a2f94..5a09cebd 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.pull.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -1,5 +1,5 @@ // Tests for the semantic-model Knowledge Catalog pull leg -// (pullKnowledgeCatalog in src/libts/semantic/deploy_knowledge_catalog.ts). +// (pullKnowledgeCatalog in src/libts/semantic/pull_kc.ts). // // pullKnowledgeCatalog is the orchestration around the pure reader: enumerate // the entry group, hydrate each semantic entry's aspect data, and reconstruct @@ -7,7 +7,7 @@ // the fake serves are produced by the real emitter, so this exercises the true // list -> hydrate -> read path end to end (an entity is re-fetched with BOTH // its semantic-entity and schema aspects). The reader's own mapping is covered -// in knowledge_catalog.read.test.ts; the focus here is the fetch SEQUENCE: +// in kc_converter.test.ts; the focus here is the fetch SEQUENCE: // aspect hydration, the --model filter, skipped entries, and ignoring foreign // entries. @@ -15,7 +15,7 @@ import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; import {ApiResult} from '../../../src/libts/gcp/api'; import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; -import {pullKnowledgeCatalog} from '../../../src/libts/semantic/deploy_knowledge_catalog'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; import {SemanticModel} from '../../../src/libts/semantic/ir'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; From da163dc4d4e29cf048346f693622c777fc00c76a Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:29:59 +0000 Subject: [PATCH 03/14] mdcode: add TODO(#278) migration notes to the converter-scaffold files Each new converter/orchestration file now carries an actionable TODO spelling out how the scaffold collapses once #278 merges, so reviewers can see the plan: - osi_converter.ts: fold loader.ts (OSI read) in, delete loader.ts, repoint importers. - kc_converter.ts: fold generateCatalogResources (KC write) in, delete knowledge_catalog.ts, repoint importers, demote the shared idOf to a local. - pull_kc.ts: rename deploy_knowledge_catalog.ts -> push_kc.ts for push_kc/pull_kc symmetry (rename only, no logic moves). --- toolbox/mdcode/src/libts/semantic/kc_converter.ts | 8 ++++++++ toolbox/mdcode/src/libts/semantic/osi_converter.ts | 7 +++++++ toolbox/mdcode/src/libts/semantic/pull_kc.ts | 7 +++++++ 3 files changed, 22 insertions(+) diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 233f7b5e..25a67c0b 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -8,6 +8,14 @@ // two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is // the KC emit code?" -> `knowledge_catalog.ts`. // +// TODO(#278): fold the KC WRITE direction in and drop the scaffold. Once +// #278 merges, move `generateCatalogResources` (and the `KcResources` type +// + emit helpers) out of `knowledge_catalog.ts` into this file, delete +// `knowledge_catalog.ts`, and repoint its importers (`deploy_knowledge_catalog.ts` +// and the emitter tests). `idOf` -- shared by both directions and re-exported +// from here only for `pull_kc` today -- becomes a plain local. Then this is +// the sole KC<->IR codec and the SCAFFOLD note above comes out. +// // The reader is the inverse of `generateCatalogResources`: it reconstructs the // IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). // `semantic-entity` / `semantic-metric` entries are grouped under their diff --git a/toolbox/mdcode/src/libts/semantic/osi_converter.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts index 61c4a99b..b8b260aa 100644 --- a/toolbox/mdcode/src/libts/semantic/osi_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -6,6 +6,13 @@ // KC push line) merges, at which point this becomes the full two-way // converter. Until then, "where is the OSI parser?" -> `loader.ts`. // +// TODO(#278): fold the OSI READ direction in and drop the scaffold. Once +// #278 merges, move `loadModels` / `fromDocument` / `loadSemanticModels` +// (and their Load* types) out of `loader.ts` into this file, delete +// `loader.ts`, and repoint its importers (`src/tool/commands.ts` and the +// load / validate / bigquery tests). Then this is the sole OSI<->IR codec +// and the SCAFFOLD note above comes out. +// // Serializing is the inverse of `loader.ts`: `loader` reads authored YAML // into the IR; this module writes the IR out as a YAML document the loader // can read back. It is the local-workspace sink for `pull` (Knowledge diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index f03edc3f..aebaa7b5 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -8,6 +8,13 @@ // types -- an entity needs BOTH its `semantic-entity` aspect and the built-in // `schema` aspect), and hands the hydrated entries to the pure reader // (`kc_converter.modelsFromCatalogResources`). +// +// TODO(#278): Layer 2 push/pull symmetry. This is the pull half over the +// pure `kc_converter` codec; the push half (`deployKnowledgeCatalog`) still +// lives in `deploy_knowledge_catalog.ts`. Once #278 merges, rename that +// file to `push_kc.ts` (repointing `src/tool/commands.ts` and its test) so +// the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no +// logic moves. import {CatalogClient, Entry} from '../gcp/dataplex'; import {SemanticModel} from './ir'; From 5b41f6429c5afa2f168c716df81e045d288efd26 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:34:51 +0000 Subject: [PATCH 04/14] mdcode: state push/pull lossiness explicitly in the user guide Reviewers asked the docs to be clear about lossless vs lossy. Both directions are lossy; say so plainly and enumerate exactly what each drops: - Push to BigQuery is lossy: captures the queryable structure (node/edge tables, measures) but not descriptive metadata; non-reducible metrics are skipped. - Push to Knowledge Catalog is lossy: stores a metadata subset (keeps 1:1/1:N as schema-join links) and drops keys, ai_context, labels, vendor SQL, M:N. - Pull is lossy: recovers even less than the catalog holds (no relationships, no deploymentTargets). A push followed by a pull does not return the original file. --- toolbox/mdcode/docs/semantic-model.md | 49 ++++++++++++++++----------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 7d3c28a2..80b7db91 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -138,6 +138,12 @@ statement runs in the right region; without that permission it falls back to BigQuery's own location inference and warns. Nothing runs under `--validate-only`; add `--print` to see the DDL. +Push to BigQuery is **lossy**: the graph captures the queryable structure — +node tables, edge tables, and measures — but not descriptive metadata +(descriptions, `ai_context`, synonyms, labels), and a metric that does not +reduce to a single MEASURE is dropped with a warning. It is a query surface, +not a copy of your model. + ### What gets created in Knowledge Catalog Each element of your model maps to one catalog resource. Every resource type @@ -155,16 +161,19 @@ An entity entry carries its columns in the `schema` aspect (name, data type, and description per field); a `schema-join` link carries the relationship detail — the paired columns and foreign-key direction — in its aspect. -> **Note — the catalog is not a full copy of your model.** By default the SQL -> expressions are **not** written to Knowledge Catalog: the published system-type -> templates do not yet carry a per-field `semantics` block or a -> `semantic-metric.expression` field, so the default push omits them (pass -> `--emit-expressions` to write them once the templates gain the fields). The -> original vendor SQL (`importedExpression` — e.g. the MAQL or Snowflake form a -> metric was imported from) is never written either. All of it stays in your -> authored document and is still used when generating BigQuery SQL. Keep your -> model document as the source of truth: a model reconstructed only from the -> catalog would come back without its SQL. +> **Note — push to Knowledge Catalog is lossy.** The catalog holds metadata, +> not a full copy of your model. It **stores** names, descriptions, data +> sources, field datatypes and roles, and 1:1 / 1:N relationships (as +> `schema-join` links). By default it does **not** store the SQL expressions: +> the published system-type templates do not yet carry a per-field `semantics` +> block or a `semantic-metric.expression` field, so the default push omits them +> (pass `--emit-expressions` to write the canonical GoogleSQL/ANSI expression +> once the templates gain the fields). It never stores entity keys, `ai_context`, +> field labels, the original vendor SQL (`importedExpression` — e.g. the MAQL or +> Snowflake form a metric was imported from), or M:N relationships. Those stay in +> your authored document (and, for the edges, in the BigQuery property graph); the +> vendor SQL and expressions are still used when generating BigQuery SQL. Keep +> your model document as the source of truth. ## Validation @@ -250,16 +259,18 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull recovers the catalog, not your authored document.** The catalog -> stores only what push wrote to it (see the note under [What gets created in -> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)), so a pulled -> document comes back without the content the catalog never held: entity keys, +> **Note — pull is lossy.** It reconstructs a model only from what push wrote +> to the catalog (see the note under [What gets created in Knowledge +> Catalog](#what-gets-created-in-knowledge-catalog)), and recovers even less +> than the catalog holds. A pulled document comes back without entity keys, > `ai_context`, field labels, the original vendor SQL (`importedExpression`), -> and relationships (the graph edges live in the BigQuery property graph, not -> the catalog). A field's *role* survives as a bare `dimension: {}` marker, but -> its detail (`is_time`, and so on) does not. Keep your authored document as the -> source of truth; treat a pulled document as a faithful copy of the catalog -> metadata, not of the original model. +> relationships (even the 1:1 / 1:N `schema-join` links push wrote — the edges +> live in the BigQuery property graph), and the `deploymentTargets` custom +> extension. A field's *role* survives as a bare `dimension: {}` marker, but its +> detail (`is_time`, and so on) does not. **A push followed by a pull does not +> return your original file** — treat a pulled document as a faithful copy of +> the catalog metadata, not of the authored model, and keep the authored +> document as the source of truth. ## Permissions From b0dcf579ef9a198a5c562c1171212a924eefc2c5 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 06:57:30 +0000 Subject: [PATCH 05/14] mdcode: recover relationships and deployment targets on pull Pull previously dropped two things push had already written to Knowledge Catalog: the model's deployment targets (stored in the semantic-model aspect) and its 1:1/1:N relationships (stored as schema-join entry links). The reader only opened per-entry aspects and pull only fetched entries, so both were silently lost even though the catalog held them. - kc_converter: read deploymentTargets back into the GOOGLE custom_extensions block, and invert schema-join links into Relationships -- endpoints resolved by data source, FK direction and columns from the join aspect. modelsFromCatalogResources grows an entryLinks argument. Relationship names come back normalized (lowercased/hyphenated): the emitter stores the name only in the link id. - pull_kc: add a second fetch pass over the entity entries via lookupEntryLinks, deduping the undirected links (each is returned from both endpoints). - Tests cover endpoint/direction recovery, name normalization, the M:N drop, deployment-target recovery, and the pull fetch+dedup path; the .pull.golden fixtures are regenerated and the docs pull note rewritten. M:N (association) relationships remain unrecovered -- push never emits them. Writer files are untouched. --- toolbox/mdcode/docs/semantic-model.md | 28 +- .../mdcode/src/libts/semantic/kc_converter.ts | 210 +++++++++++--- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 76 +++++- .../sales_bq_graph_target.pull.golden.yaml | 3 + .../star_orders_customer.pull.golden.yaml | 11 + .../fixtures/tpcds_date_edge.pull.golden.yaml | 29 ++ .../tests/libts/semantic/kc_converter.test.ts | 257 +++++++++++++----- .../tests/libts/semantic/pull_kc.test.ts | 107 +++++++- 8 files changed, 583 insertions(+), 138 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 80b7db91..ec67acbd 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -259,18 +259,22 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull is lossy.** It reconstructs a model only from what push wrote -> to the catalog (see the note under [What gets created in Knowledge -> Catalog](#what-gets-created-in-knowledge-catalog)), and recovers even less -> than the catalog holds. A pulled document comes back without entity keys, -> `ai_context`, field labels, the original vendor SQL (`importedExpression`), -> relationships (even the 1:1 / 1:N `schema-join` links push wrote — the edges -> live in the BigQuery property graph), and the `deploymentTargets` custom -> extension. A field's *role* survives as a bare `dimension: {}` marker, but its -> detail (`is_time`, and so on) does not. **A push followed by a pull does not -> return your original file** — treat a pulled document as a faithful copy of -> the catalog metadata, not of the authored model, and keep the authored -> document as the source of truth. +> **Note — pull is lossy.** It reconstructs a model from what push wrote to the +> catalog (see the note under [What gets created in Knowledge +> Catalog](#what-gets-created-in-knowledge-catalog)), so it recovers the model +> structure — entities and fields, metrics, 1:1 / 1:N relationships (from the +> `schema-join` links), and the deployment targets. It does **not** recover the +> content the catalog never held: entity keys, `ai_context`, field labels, the +> original vendor SQL (`importedExpression`), and M:N relationships (whose edge +> lives only in the BigQuery property graph). Some recovered content also comes +> back normalized rather than verbatim: relationship names are +> lowercased/hyphenated (the catalog stores the name only in the link id), a +> field's *role* survives as a bare `dimension: {}` marker without its detail +> (`is_time`, and so on), and a metric authored without a datatype comes back as +> an explicit `Decimal`. **A push followed by a pull does not return your original +> file** — treat a pulled document as a faithful copy of the catalog metadata, +> not of the authored model, and keep the authored document as the source of +> truth. ## Permissions diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 25a67c0b..89dd0ad5 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -1,39 +1,47 @@ // Knowledge Catalog <-> Semantic Model IR converter. // // SCAFFOLD (naming for the end state): this file currently holds only the -// READ direction -- Knowledge Catalog entries -> IR (`modelsFromCatalogResources` -// and its helpers). The WRITE direction (IR -> KC entries, -// `generateCatalogResources`) still lives in `knowledge_catalog.ts` and moves -// here once PR4 (the KC push line) merges, at which point this becomes the full -// two-way converter and `knowledge_catalog.ts` goes away. Until then, "where is -// the KC emit code?" -> `knowledge_catalog.ts`. +// READ direction -- Knowledge Catalog entries -> IR +// (`modelsFromCatalogResources` and its helpers). The WRITE direction (IR -> KC +// entries, `generateCatalogResources`) still lives in `knowledge_catalog.ts` +// and moves here once PR4 (the KC push line) merges, at which point this +// becomes the full two-way converter and `knowledge_catalog.ts` goes away. +// Until then, "where is the KC emit code?" -> `knowledge_catalog.ts`. // // TODO(#278): fold the KC WRITE direction in and drop the scaffold. Once // #278 merges, move `generateCatalogResources` (and the `KcResources` type // + emit helpers) out of `knowledge_catalog.ts` into this file, delete -// `knowledge_catalog.ts`, and repoint its importers (`deploy_knowledge_catalog.ts` -// and the emitter tests). `idOf` -- shared by both directions and re-exported -// from here only for `pull_kc` today -- becomes a plain local. Then this is -// the sole KC<->IR codec and the SCAFFOLD note above comes out. +// `knowledge_catalog.ts`, and repoint its importers +// (`deploy_knowledge_catalog.ts` and the emitter tests). `idOf` -- shared by +// both directions and re-exported from here only for `pull_kc` today -- becomes +// a plain local. Then this is the sole KC<->IR codec and the SCAFFOLD note +// above comes out. // // The reader is the inverse of `generateCatalogResources`: it reconstructs the // IR from the entries a pull hydrated (see `pull_kc.pullKnowledgeCatalog`). // `semantic-entity` / `semantic-metric` entries are grouped under their -// `semantic-model` anchor via `parentEntry`; entries of other types are ignored. -// Resources are matched by type-name SUFFIX, so a reader need not know which -// system-type project/location the emitter used. +// `semantic-model` anchor via `parentEntry`; entries of other types are +// ignored. Resources are matched by type-name SUFFIX, so a reader need not know +// which system-type project/location the emitter used. // -// Fidelity is bounded by what the emitter persisted, so this read is the inverse -// of the WRITE, not of the authored document. It recovers names, descriptions, -// data sources, field datatypes (via the schema aspect) and DIMENSION roles, -// field/metric expressions, and each metric's attach entity (re-derived from its -// expression, as the loader does). It cannot recover what the emitter does not -// write: entity keys/unique keys, `ai_context`, field labels, `importedDialect`, -// `custom_extensions`, and relationships (the graph edges live in the BigQuery -// property graph, not the catalog). - -import type {Entry} from '../gcp/dataplex'; -import {DataType, Entity, Field, Metric, SemanticModel} from './ir'; +// Fidelity is bounded by what the emitter persisted, so this read is the +// inverse of the WRITE, not of the authored document. It recovers names, +// descriptions, data sources, field datatypes (via the schema aspect) and +// DIMENSION roles, field/metric expressions, each metric's attach entity +// (re-derived from its expression, as the loader does), the model's deployment +// targets (from the semantic-model aspect, back into the GOOGLE +// `custom_extensions` block), and 1:1 / 1:N relationships (from the +// `schema-join` entry links a pull fetched -- see +// `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what +// the emitter does not write: entity keys/unique keys, `ai_context`, field +// labels, `importedDialect`, and many-to-many (association) relationships +// (whose edge lives only in the BigQuery property graph). Relationship NAMES +// come back normalized (lowercased/hyphenated), since the emitter encodes the +// name only in the link id (via `linkSlug`), not in the join aspect. + +import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; + +import {CustomExtension, DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; import {referencedEntityNames} from './sql_expr_utils'; export interface ReadResult { @@ -48,8 +56,14 @@ export interface ReadResult { * an orphaned child, an entry missing its aspect data). Entries must already be * hydrated with their `semantic-*` (and, for entities, `schema`) aspect data; a * BASIC list omits aspect data, so the puller re-fetches each entry first. + * + * `entryLinks` are the `schema-join` links a pull fetched for the group (see + * `pull_kc`); each link between two of a model's entity entries is + * reconstructed as a 1:1 / 1:N relationship. Pass `[]` (the default) to + * reconstruct entities and metrics only. */ -export function modelsFromCatalogResources(entries: Entry[]): ReadResult { +export function modelsFromCatalogResources( + entries: Entry[], entryLinks: EntryLink[] = []): ReadResult { const warnings: string[] = []; const anchors = entries.filter(e => semanticType(e) === 'semantic-model'); @@ -75,17 +89,26 @@ export function modelsFromCatalogResources(entries: Entry[]): ReadResult { const models = anchors.map(anchor => { const name = anchor.entrySource?.displayName ?? idOf(anchor.name); - const entities = childrenOf(anchor.name, entityEntries) - .map(e => readEntity(e, warnings)); + const entityEntriesForModel = childrenOf(anchor.name, entityEntries); + const entities = entityEntriesForModel.map(e => readEntity(e, warnings)); const entityNames = entities.map(e => e.name); const metrics = childrenOf(anchor.name, metricEntries) .map(e => readMetric(e, entityNames, warnings)); - // Relationships are not published to the catalog (see the file header), so - // a reconstructed model always has an empty edge set. - const model: SemanticModel = {name, entities, relationships: [], metrics}; + // Relationships come from the schema-join entry links whose two endpoints + // are both this model's entity entries (M:N edges were never published -- + // they live only in the BigQuery property graph -- so stay absent here). + const relationships = readRelationships( + entryLinks, name, new Set(entityEntriesForModel.map(e => e.name)), + dataSourceIndex(entities), warnings); + + const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; if (description !== undefined) model.description = description; + // Deployment targets ride back in the same GOOGLE custom_extensions block + // the author wrote them in (the inverse of the emitter's modelAspectData). + const targets = readDeploymentTargets(anchor); + if (targets) model.customExtensions = [targets]; return model; }); @@ -248,13 +271,21 @@ function semanticType(entry: Entry): 'semantic-model'|'semantic-entity'| } -// The `data` payload of an entry's aspect of the given bare type, matched by -// the aspect key's `.` suffix or the aspectType's `/aspectTypes/` -// suffix (robust to whichever system-type project/location the emitter used). -// Returns an empty object when the aspect is absent. +// The `data` payload of an entry's aspect of the given bare type. See +// aspectDataOf; entries and entry links carry aspects in the same shape. function aspectData(entry: Entry, type: string): Record { - const aspects = entry.aspects ?? {}; - for (const [key, aspect] of Object.entries(aspects)) { + return aspectDataOf(entry.aspects, type); +} + + +// The `data` payload of an aspect of the given bare type from an aspect map, +// matched by the aspect key's `.` suffix or the aspectType's +// `/aspectTypes/` suffix (robust to whichever system-type +// project/location the emitter used). Returns an empty object when the aspect +// is absent. +function aspectDataOf(aspects: Record|undefined, type: string): + Record { + for (const [key, aspect] of Object.entries(aspects ?? {})) { if (key.endsWith(`.${type}`) || aspect.aspectType?.endsWith(`/aspectTypes/${type}`)) { return aspect.data ?? {}; @@ -264,6 +295,113 @@ function aspectData(entry: Entry, type: string): Record { } +// Recovers the model's deployment targets from its semantic-model aspect back +// into the GOOGLE custom_extension the author declared them in (the inverse of +// the emitter's modelAspectData). Returns undefined when the model has none. +function readDeploymentTargets(anchor: Entry): CustomExtension|undefined { + const targets = + asArray(aspectData(anchor, 'semantic-model').deploymentTargets) + .filter((t): t is string => typeof t === 'string' && t !== ''); + if (!targets.length) return undefined; + return { + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }; +} + + +// Indexes reconstructed entities by their SQL data source, so a schema-join +// aspect (which names each side by its table, not the entity) can resolve its +// endpoints back to entity names. +function dataSourceIndex(entities: Entity[]): Map { + const index = new Map(); + for (const entity of entities) { + const dataSource = (entity.dataSource ?? '').trim(); + if (dataSource) index.set(dataSource, entity.name); + } + return index; +} + + +// Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link +// whose two endpoints are both this model's entity entries becomes one +// direct-FK relationship. The aspect encodes direction (`source` is the +// foreign-key side) and the paired join columns; the link id encodes the +// (normalized) name. Links are deduped by name -- schema-join is undirected, so +// a per-entry lookup can return the same link once from each endpoint. +function readRelationships( + links: EntryLink[], modelName: string, entityEntryNames: Set, + dataSourceToEntity: Map, + warnings: string[]): Relationship[] { + const prefix = linkNamePrefix(modelName); + const seen = new Set(); + const out: Relationship[] = []; + for (const link of links) { + if (!link.entryLinkType?.endsWith('/entryLinkTypes/schema-join')) continue; + const refs = link.entryReferences ?? []; + // Only a link whose BOTH endpoints are this model's entities belongs here. + if (refs.length !== 2 || !refs.every(r => entityEntryNames.has(r.name))) { + continue; + } + if (link.name) { + if (seen.has(link.name)) continue; + seen.add(link.name); + } + const join = asArray(aspectDataOf(link.aspects, 'schema-join').joins)[0]; + if (!join) { + warnings.push( + `entry link '${link.name}': no schema-join aspect data; the ` + + `relationship is skipped`); + continue; + } + const source = dataSourceToEntity.get((join.source?.name ?? '').trim()); + const destination = + dataSourceToEntity.get((join.target?.name ?? '').trim()); + if (!source || !destination) { + warnings.push( + `entry link '${link.name}': a join endpoint table does not match a ` + + `reconstructed entity; the relationship is skipped`); + continue; + } + const rel: Relationship = { + name: relationshipName(link.name, prefix), + source: {entity: source, columns: asArray(join.source?.fields)}, + destination: {entity: destination, columns: asArray(join.target?.fields)}, + }; + if (join.description !== undefined) rel.description = join.description; + out.push(rel); + } + return out; +} + + +// Best-effort recovery of the relationship name from the link id, which the +// emitter built as linkSlug(`-`) (lowercased/hyphenated -- see +// knowledge_catalog.ts). Strips the model prefix when present; the name comes +// back normalized, never verbatim. +function relationshipName( + linkName: string|undefined, modelPrefix: string): string { + const id = idOf(linkName ?? ''); + if (modelPrefix && id.startsWith(`${modelPrefix}-`)) { + const rest = id.slice(modelPrefix.length + 1); + if (rest) return rest; + } + return id; +} + + +// Mirrors the model-name portion of the emitter's linkSlug so the prefix a link +// id was built with can be stripped. Kept local rather than imported: this +// read-side module must not depend on the write-side emitter. +function linkNamePrefix(modelName: string): string { + return modelName.toLowerCase() + .replace(/[^a-z0-9-]+/g, '-') + .replace(/-+/g, '-') + .replace(/^[^a-z]+/, '') + .replace(/^-+|-+$/g, ''); +} + + // The id segment of a full entry resource name (after the last '/'). export function idOf(name: string): string { return name.split('/').pop() ?? name; diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index aebaa7b5..24d338c3 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -16,7 +16,8 @@ // the orchestration layer reads as `push_kc` / `pull_kc`. Rename only -- no // logic moves. -import {CatalogClient, Entry} from '../gcp/dataplex'; +import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; + import {SemanticModel} from './ir'; import {idOf, modelsFromCatalogResources} from './kc_converter'; @@ -65,9 +66,8 @@ export async function pullKnowledgeCatalog( if (!scoped.length) { return { models: [], - warnings: [ - `no semantic model named '${opts.model}' found in ${destination}` - ], + warnings: + [`no semantic model named '${opts.model}' found in ${destination}`], }; } } @@ -93,7 +93,45 @@ export async function pullKnowledgeCatalog( warnings.push(r.warning); } - const read = modelsFromCatalogResources(hydrated); + // Second fetch pass: relationships are schema-join entry links, which the + // entry list/lookup does not return. The catalog exposes links only per + // referenced entry (:lookupEntryLinks), so fan out over the entity entries + // and dedup by link name -- schema-join is undirected, so each link comes + // back once from each of its two endpoints. + const entityEntries = hydrated.filter( + e => e.entryType?.endsWith('/entryTypes/semantic-entity')); + const linkResults = + await mapConcurrent(entityEntries, HYDRATE_CONCURRENCY, async entry => { + const linkType = schemaJoinLinkType(entry.entryType); + const res = await cat.lookupEntryLinks(opts.project, opts.location, { + entry: entry.name, + entryLinkTypes: linkType ? [linkType] : undefined, + }); + if (res.status !== 200 || !res.result) { + return { + warning: `failed to fetch entry links for '${entry.name}' (status ${ + res.status}); relationships may be incomplete`, + }; + } + return {links: res.result}; + }); + + const seenLinks = new Set(); + const entryLinks: EntryLink[] = []; + for (const r of linkResults) { + if (r.warning) { + warnings.push(r.warning); + continue; + } + for (const link of r.links ?? []) { + const key = link.name ?? ''; + if (key && seenLinks.has(key)) continue; + if (key) seenLinks.add(key); + entryLinks.push(link); + } + } + + const read = modelsFromCatalogResources(hydrated, entryLinks); warnings.push(...read.warnings); // Defense in depth: keep only the requested model even if the reader surfaced @@ -136,6 +174,19 @@ function semanticAspectTypes(entryType: string): string[]|undefined { } +// The schema-join entry link type resource, derived from an entity's entryType +// base (the link type is the parallel resource in the same project/location the +// emitter referenced). Used to filter :lookupEntryLinks to just the +// relationship links. Returns undefined for an entryType with no recognizable +// base. +function schemaJoinLinkType(entryType: string): string|undefined { + const marker = '/entryTypes/'; + const idx = entryType?.indexOf(marker) ?? -1; + if (idx < 0) return undefined; + return `${entryType.slice(0, idx)}/entryLinkTypes/schema-join`; +} + + // Restricts hydration targets to a single model: the semantic-model anchor // whose name (entrySource.displayName, else the entry id) matches `model`, plus // every child entry whose parentEntry is that anchor. Uses only list-level @@ -146,13 +197,14 @@ function scopeToModel( model: string): {entry: Entry; aspectTypes: string[]}[] { const isAnchor = (t: {entry: Entry}) => !!t.entry.entryType?.endsWith('/entryTypes/semantic-model'); - const allAnchorNames = new Set(targets.filter(isAnchor).map(t => t.entry.name)); - const matchedAnchorNames = new Set( - targets.filter(isAnchor) - .filter( - t => (t.entry.entrySource?.displayName ?? idOf(t.entry.name)) === - model) - .map(t => t.entry.name)); + const allAnchorNames = + new Set(targets.filter(isAnchor).map(t => t.entry.name)); + const matchedAnchorNames = + new Set(targets.filter(isAnchor) + .filter( + t => (t.entry.entrySource?.displayName ?? + idOf(t.entry.name)) === model) + .map(t => t.entry.name)); if (!matchedAnchorNames.size) return []; // Mirror the reader's childrenOf (knowledge_catalog.ts): when the group holds // exactly one anchor, a child whose parentEntry resolves to no anchor (e.g. a diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml index f9e42580..06dd8bdc 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -2,6 +2,9 @@ version: 0.2.0.dev0 semantic_model: - name: sales + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/demo/datasets/sales/propertyGraphs/sales_graph"]}' datasets: - name: orders source: demo.sales.orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml index bf7cfc8d..d7a9cbe1 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -3,6 +3,9 @@ version: 0.2.0.dev0 semantic_model: - name: sales description: Sales orders with customer attributes + custom_extensions: + - vendor_name: GOOGLE + data: '{"deploymentTargets":["//bigquery.googleapis.com/projects/sqlgen-testing/datasets/demo/propertyGraphs/sales"]}' datasets: - name: orders source: samples.tpch.orders @@ -44,6 +47,14 @@ semantic_model: - dialect: BIGQUERY expression: c_name description: Customer name + relationships: + - name: orders-to-customer + from: orders + to: customer + from_columns: + - o_custkey + to_columns: + - c_custkey metrics: - name: total_revenue expression: diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml index 7437f8ac..1900fe12 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -145,3 +145,32 @@ semantic_model: - name: date_dim source: sqlgen-testing.demo.date_dim description: Date dimension with calendar attributes + relationships: + - name: store-sales-to-date-dim + from: store_sales + to: date_dim + from_columns: + - ss_sold_date_sk + to_columns: + - ss_sold_date_sk + - name: store-sales-to-customer + from: store_sales + to: customer + from_columns: + - ss_customer_sk + to_columns: + - c_customer_sk + - name: store-sales-to-item + from: store_sales + to: item + from_columns: + - ss_item_sk + to_columns: + - i_item_sk + - name: store-sales-to-store + from: store_sales + to: store + from_columns: + - ss_store_sk + to_columns: + - s_store_sk diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index d5ecb145..0e2f0eb6 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -2,22 +2,26 @@ // (modelsFromCatalogResources in src/libts/semantic/kc_converter.ts). // // The reader is the inverse of the emitter (generateCatalogResources). The -// central guarantee is an emitter -> reader round trip: emit a model's entries, -// read them back, and get an IR equal to the source WHERE the emitter is -// lossless. The write drops content by design (entity keys, ai_context, field -// labels, importedDialect, relationships -- see the emitter header), so the -// expected read-back is the source model with exactly those fields cleared. +// central guarantee is an emitter -> reader round trip: emit a model's entries +// AND entry links, read them back, and get an IR equal to the source WHERE the +// emitter is lossless. The write drops content by design (entity keys, +// ai_context, field labels, importedDialect, and many-to-many relationships -- +// see the emitter header), so the expected read-back is the source model with +// exactly those fields cleared. 1:1 / 1:N relationships and deployment targets +// DO round-trip (via schema-join links and the semantic-model aspect), except +// that relationship names come back normalized (lowercased/hyphenated). // Targeted tests pin the mapping details a round trip cannot isolate (the // dataType inverse, the DIMENSION role, resource-URI parsing, metric attach -// re-derivation, and parent/anchor grouping). +// re-derivation, relationship endpoint/direction recovery, and parent/anchor +// grouping). import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; import {serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -28,11 +32,11 @@ const OPTS = { entryGroup: 'eg' }; -// Emits a model to entries and reads it straight back. +// Emits a model to entries + entry links and reads it straight back. function roundTrip(model: SemanticModel): {models: SemanticModel[]; warnings: string[]} { - const {entries} = generateCatalogResources(model, OPTS); - return modelsFromCatalogResources(entries); + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources(entries, entryLinks); } @@ -75,6 +79,120 @@ describe('emitter -> reader round trip (lossless slice)', () => { }); +describe('relationship recovery (schema-join links -> IR)', () => { + // orders.o_custkey (the foreign-key side) references customer.c_custkey. + const twoEntities: Entity[] = [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ]; + + test( + 'a 1:N relationship recovers its endpoints, direction, and columns', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: + 'places', // already link-slug-safe, so it round-trips exactly + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const rels = roundTrip(model).models[0].relationships; + expect(rels).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'a relationship name comes back normalized (lowercased/hyphenated)', + () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'Places_Order', // mixed case + underscore -> normalized on + // read + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('places-order'); + }); + + test('a many-to-many (association) relationship is not recovered', () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'enrolls', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + association: { + dataSource: 'p.d.junction', + keys: ['id'], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + }, + }], + metrics: [], + }; + // The emitter never publishes M:N, so no schema-join link exists to read. + expect(roundTrip(model).models[0].relationships).toEqual([]); + }); +}); + + +describe( + 'deployment-target recovery (semantic-model aspect -> custom_extensions)', + () => { + test('the GOOGLE deployment targets ride back verbatim', () => { + const uri = + '//bigquery.googleapis.com/projects/p/datasets/d/propertyGraphs/g'; + const data = JSON.stringify({deploymentTargets: [uri]}); + const model: SemanticModel = { + name: 'sales', + customExtensions: [{vendorName: 'GOOGLE', data}], + entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toEqual([ + {vendorName: 'GOOGLE', data} + ]); + }); + + test( + 'a model with no deployment targets recovers no custom_extensions', + () => { + const model: SemanticModel = { + name: 'sales', + entities: + [{name: 'e', dataSource: 'p.d.t', keys: [], fields: []}], + relationships: [], + metrics: [], + }; + expect(roundTrip(model).models[0].customExtensions).toBeUndefined(); + }); + }); + + describe('dataType inverse (schema aspect -> IR type)', () => { // Emit a one-field model of each IR type, read it back, and check the field's // reconstructed type. String and Opaque both emit dataType STRING; String @@ -246,21 +364,22 @@ describe('anchor / parent grouping', () => { describe('metric expression referencing no known entity', () => { - test('warns that the metric may be unplaceable and leaves it unattached', - () => { - const model: SemanticModel = { - name: 'm', - entities: - [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], - relationships: [], - // References `widgets`, which is not an entity of this model. - metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], - }; - const {models, warnings} = roundTrip(model); - expect(models[0].metrics[0].entity).toBeUndefined(); - expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) - .toBe(true); - }); + test( + 'warns that the metric may be unplaceable and leaves it unattached', + () => { + const model: SemanticModel = { + name: 'm', + entities: + [{name: 'orders', dataSource: 'p.d.o', keys: [], fields: []}], + relationships: [], + // References `widgets`, which is not an entity of this model. + metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].metrics[0].entity).toBeUndefined(); + expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) + .toBe(true); + }); }); @@ -268,48 +387,52 @@ describe('metric expression referencing no known entity', () => { // // The round trip above proves the reader inverts the emitter in memory; this // pins the reviewable artifact. For each corpus fixture it reads the committed -// emitter golden (`.knowledge_catalog.golden.json` -- the exact entries -// a push produced) back through the reader and serializes the reconstructed IR -// to `.pull.golden.yaml`. Open that next to the fixture's -// `.osi.golden.yaml` to see, as whole files, what a Knowledge Catalog round trip -// preserves and what it drops (keys, ai_context, labels, relationships). +// emitter golden (`.knowledge_catalog.golden.json` -- the exact +// entries and entry links a push produced) back through the reader and +// serializes the reconstructed IR to `.pull.golden.yaml`. Open that +// next to the fixture's `.osi.golden.yaml` to see, as whole files, what a +// Knowledge Catalog round trip preserves (including 1:1 / 1:N relationships and +// deployment targets) and what it drops (keys, ai_context, labels, vendor SQL, +// M:N relationships). // // Regenerate after an intentional reader/serializer change: // UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts -describe('golden pull: each corpus KC golden reconstructs to its exact YAML', - () => { - const CORPUS = [ - 'sales_bq_graph_target.yaml', - 'star_orders_customer.yaml', - 'tpcds_date_edge.yaml', - ]; - const kcGoldenPath = (fixture: string) => path.join( - FIXTURES, - fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); - const pullGoldenPath = (fixture: string) => - path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); - - for (const fixture of CORPUS) { - test(fixture, () => { - const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); - const {models, warnings} = modelsFromCatalogResources(kc.entries); - // Reader warnings ride along as YAML comments so the golden shows - // the full outcome, not just the recovered document. - const header = warnings.length ? - warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : - '# (no warnings)\n'; - const actual = - header + models.map(m => serializeModel(m).yaml).join('---\n'); - const golden = pullGoldenPath(fixture); - if (process.env.UPDATE_GOLDENS) { - fs.writeFileSync(golden, actual); - return; - } - if (!fs.existsSync(golden)) { - throw new Error(`missing golden ${ - path.basename(golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); - } - expect(actual).toBe(fs.readFileSync(golden, 'utf8')); - }); - } - }); +describe( + 'golden pull: each corpus KC golden reconstructs to its exact YAML', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + const kcGoldenPath = (fixture: string) => path.join( + FIXTURES, + fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + const pullGoldenPath = (fixture: string) => + path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); + + for (const fixture of CORPUS) { + test(fixture, () => { + const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const {models, warnings} = + modelsFromCatalogResources(kc.entries, kc.entryLinks ?? []); + // Reader warnings ride along as YAML comments so the golden shows + // the full outcome, not just the recovered document. + const header = warnings.length ? + warnings.map((w: string) => `# warning: ${w}`).join('\n') + '\n' : + '# (no warnings)\n'; + const actual = + header + models.map(m => serializeModel(m).yaml).join('---\n'); + const golden = pullGoldenPath(fixture); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, actual); + return; + } + if (!fs.existsSync(golden)) { + throw new Error(`missing golden ${ + path.basename( + golden)} \u2014 run UPDATE_GOLDENS=1 to create it`); + } + expect(actual).toBe(fs.readFileSync(golden, 'utf8')); + }); + } + }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index 5a09cebd..d5d22d42 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -6,18 +6,19 @@ // the IR. The catalog client is stubbed so no network call is made. The entries // the fake serves are produced by the real emitter, so this exercises the true // list -> hydrate -> read path end to end (an entity is re-fetched with BOTH -// its semantic-entity and schema aspects). The reader's own mapping is covered -// in kc_converter.test.ts; the focus here is the fetch SEQUENCE: -// aspect hydration, the --model filter, skipped entries, and ignoring foreign -// entries. +// its semantic-entity and schema aspects, and a second pass fetches each +// entity's schema-join links). The reader's own mapping is covered in +// kc_converter.test.ts; the focus here is the fetch SEQUENCE: aspect hydration, +// the relationship-link fetch (per-entry, deduped across endpoints), the +// --model filter, skipped entries, and ignoring foreign entries. import {afterEach, describe, expect, mock, spyOn, test} from 'bun:test'; import {ApiResult} from '../../../src/libts/gcp/api'; -import {CatalogClient, Entry} from '../../../src/libts/gcp/dataplex'; -import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; +import {CatalogClient, Entry, EntryLink} from '../../../src/libts/gcp/dataplex'; import {SemanticModel} from '../../../src/libts/semantic/ir'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {pullKnowledgeCatalog} from '../../../src/libts/semantic/pull_kc'; const OPTS = { project: 'dest', @@ -60,8 +61,13 @@ function err(status: number, message: string): ApiResult { // Stubs listEntries (yields `listed`) and lookupEntry (serves `served` by name, // or 404s an unknown name). `lookupFail` forces a failure for one entry name. -function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): - {list: any; lookup: any} { +// `links` are served by lookupEntryLinks per referenced entry (so an undirected +// link comes back once from each endpoint, exercising the puller's dedup); +// `linkFail` forces a link-lookup failure for one entry name. +function stubClient( + listed: Entry[], served: Entry[], lookupFail?: string, + links: EntryLink[] = [], + linkFail?: string): {list: any; lookup: any; lookupLinks: any} { const byName = new Map(served.map(e => [e.name, e])); const list = spyOn(CatalogClient.prototype, 'listEntries') .mockImplementation(async function*() { @@ -73,7 +79,16 @@ function stubClient(listed: Entry[], served: Entry[], lookupFail?: string): const e = byName.get(name); return e ? ok(e) : err(404, 'not found'); }); - return {list, lookup}; + const lookupLinks = + spyOn(CatalogClient.prototype, 'lookupEntryLinks') + .mockImplementation(async (_p: any, _l: any, opts: any) => { + if (opts.entry === linkFail) return err(500, 'boom'); + const forEntry = links.filter( + l => + (l.entryReferences ?? []).some(r => r.name === opts.entry)); + return ok(forEntry); + }); + return {list, lookup, lookupLinks}; } afterEach(() => { @@ -121,6 +136,76 @@ describe('pullKnowledgeCatalog: happy path', () => { }); +describe('pullKnowledgeCatalog: relationship links', () => { + // A 1:N model: orders.o_custkey (the FK side) references customer.c_custkey. + const SALES_REL: SemanticModel = { + name: 'sales', + entities: [ + { + name: 'orders', + dataSource: 'demo.sales.orders', + keys: [], + fields: [{name: 'o_custkey', expression: 'orders.o_custkey'}], + }, + { + name: 'customer', + dataSource: 'demo.sales.customer', + keys: [], + fields: [{name: 'c_custkey', expression: 'customer.c_custkey'}], + }, + ], + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + + test( + 'a schema-join link reconstructs into a relationship exactly once', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const {lookupLinks} = + stubClient(entries, entries, undefined, entryLinks); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + expect(warnings).toHaveLength(0); + // Links are fetched per entity (2 entities), and the single undirected + // link -- returned from both endpoints -- is deduped to one edge. + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + + test( + 'a failed link lookup on one endpoint still recovers the edge from the ' + + 'other, and warns', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + const ordersEntry = + entries.find(e => (e.entrySource?.displayName) === 'orders')!; + // Fail the link lookup for the orders entity; the link still comes back + // from the customer endpoint. + stubClient(entries, entries, undefined, entryLinks, ordersEntry.name); + + const cat = new CatalogClient({} as any); + const {models, warnings} = await pullKnowledgeCatalog(cat, OPTS); + + expect(models[0].relationships.map(r => r.name)).toEqual(['places']); + expect(warnings.some( + w => /failed to fetch entry links/i.test(w) && + w.includes(ordersEntry.name))) + .toBe(true); + }); +}); + + describe('pullKnowledgeCatalog: filtering and robustness', () => { test('--model keeps only the named model', async () => { const other: SemanticModel = { @@ -215,8 +300,8 @@ describe('pullKnowledgeCatalog: filtering and robustness', () => { // drops a child a full pull returns). const metricEntry = entries.find(e => e.entryType.endsWith('/semantic-metric'))!; - metricEntry.parentEntry = - metricEntry.parentEntry!.replace('projects/dest/', 'projects/12345/'); + metricEntry.parentEntry = metricEntry.parentEntry!.replace( + 'projects/dest/', 'projects/12345/'); const {lookup} = stubClient(entries, entries); const cat = new CatalogClient({} as any); From e1982e6d79f224946320d4163bce05be2d481a18 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:10:56 +0000 Subject: [PATCH 06/14] mdcode: resolve pull relationship endpoints via entryReferences Address code-review findings on the gap-3 pull leg: - Resolve schema-join endpoints from the link's entryReferences, matched by entry id, instead of a dataSource->entity index. The id is unique per entity (fixes two entities sharing a table collapsing last-wins) and is stable across the project-number/id normalization lookupEntry applies to entries but lookupEntryLinks does not apply to link references (fixes relationships silently dropping on the live path). The schema-join aspect is now used only for FK direction + join columns; undecidable direction keeps the reference order and warns rather than dropping the edge. - Dedup entry links by a sorted endpoint-pair key when a link has no name (shared linkDedupKey, reused by pull_kc) so a nameless link returned from both endpoints is not counted twice. - Rewrite the pull 'lossy' note in the user guide as recovered-exactly / recovered-but-normalized / not-recovered bullets. Tests: shared-table endpoints, un-normalized project-number references, prefix-stripping across tricky model names, and nameless-link dedup. --- toolbox/mdcode/docs/semantic-model.md | 47 ++++--- .../mdcode/src/libts/semantic/kc_converter.ts | 111 ++++++++++------ toolbox/mdcode/src/libts/semantic/pull_kc.ts | 10 +- .../tests/libts/semantic/kc_converter.test.ts | 123 ++++++++++++++++++ .../tests/libts/semantic/pull_kc.test.ts | 19 +++ 5 files changed, 250 insertions(+), 60 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index ec67acbd..b3a1fcbb 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -259,22 +259,37 @@ Pull writes with the same last-write-wins policy as the core pull: a model that already exists locally is overwritten in place, and a local-only document (one with no matching catalog entry) is left untouched — pull never deletes. -> **Note — pull is lossy.** It reconstructs a model from what push wrote to the -> catalog (see the note under [What gets created in Knowledge -> Catalog](#what-gets-created-in-knowledge-catalog)), so it recovers the model -> structure — entities and fields, metrics, 1:1 / 1:N relationships (from the -> `schema-join` links), and the deployment targets. It does **not** recover the -> content the catalog never held: entity keys, `ai_context`, field labels, the -> original vendor SQL (`importedExpression`), and M:N relationships (whose edge -> lives only in the BigQuery property graph). Some recovered content also comes -> back normalized rather than verbatim: relationship names are -> lowercased/hyphenated (the catalog stores the name only in the link id), a -> field's *role* survives as a bare `dimension: {}` marker without its detail -> (`is_time`, and so on), and a metric authored without a datatype comes back as -> an explicit `Decimal`. **A push followed by a pull does not return your original -> file** — treat a pulled document as a faithful copy of the catalog metadata, -> not of the authored model, and keep the authored document as the source of -> truth. +> **Note — pull reconstructs what the catalog holds, not your original file.** +> Pull can only recover what push wrote (see the note under [What gets created in +> Knowledge Catalog](#what-gets-created-in-knowledge-catalog)). What that means in +> practice: +> +> **Recovered exactly** — these come back as authored: +> - Model structure: the model, its entities, and each entity's fields. +> - Field data source and data type; the field expression. +> - Metrics: name, expression, data type, and attach entity. +> - 1:1 / 1:N relationships: endpoints, foreign-key direction, and join columns +> (from the `schema-join` links). +> - Deployment targets. +> +> **Recovered, but normalized** — the content survives, the form changes: +> - Relationship *names* come back lowercased/hyphenated (the catalog stores the +> name only in the link id, e.g. `Places Order` → `places-order`). +> - A field marked as a dimension comes back as a bare `dimension: {}` marker, +> without its detail (`is_time`, and so on). +> - A metric authored with no data type comes back as an explicit `Decimal` +> (push must write a type, and defaults it to `NUMERIC`). +> +> **Not recovered** — push never wrote these, so pull cannot return them: +> - Entity keys / unique keys. +> - `ai_context`. +> - Field labels. +> - The original vendor SQL (`importedExpression`). +> - M:N relationships (the edge lives only in the BigQuery property graph). +> +> **So: a push followed by a pull does not return your original file.** Treat a +> pulled document as a faithful copy of the catalog metadata, not of the authored +> model, and keep the authored document as the source of truth. ## Permissions diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 89dd0ad5..3600d781 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -95,12 +95,21 @@ export function modelsFromCatalogResources( const metrics = childrenOf(anchor.name, metricEntries) .map(e => readMetric(e, entityNames, warnings)); + // Index this model's entities by their entry id, so a schema-join link can + // resolve its endpoints from the link's entryReferences. The id segment is + // stable across the project-number/id normalization that lookupEntry + // applies to entries but lookupEntryLinks does not apply to link + // references, so matching on it (not the full resource name) keeps + // live-fetched links resolvable. + const entityByEntryId = new Map(); + entityEntriesForModel.forEach( + (e, i) => entityByEntryId.set(idOf(e.name), entities[i])); + // Relationships come from the schema-join entry links whose two endpoints // are both this model's entity entries (M:N edges were never published -- // they live only in the BigQuery property graph -- so stay absent here). - const relationships = readRelationships( - entryLinks, name, new Set(entityEntriesForModel.map(e => e.name)), - dataSourceIndex(entities), warnings); + const relationships = + readRelationships(entryLinks, name, entityByEntryId, warnings); const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; @@ -310,28 +319,22 @@ function readDeploymentTargets(anchor: Entry): CustomExtension|undefined { } -// Indexes reconstructed entities by their SQL data source, so a schema-join -// aspect (which names each side by its table, not the entity) can resolve its -// endpoints back to entity names. -function dataSourceIndex(entities: Entity[]): Map { - const index = new Map(); - for (const entity of entities) { - const dataSource = (entity.dataSource ?? '').trim(); - if (dataSource) index.set(dataSource, entity.name); - } - return index; -} - - // Inverts schemaJoinAspectData (knowledge_catalog.ts): each schema-join link // whose two endpoints are both this model's entity entries becomes one -// direct-FK relationship. The aspect encodes direction (`source` is the -// foreign-key side) and the paired join columns; the link id encodes the -// (normalized) name. Links are deduped by name -- schema-join is undirected, so -// a per-entry lookup can return the same link once from each endpoint. +// direct-FK relationship. +// +// Endpoint identity comes from the link's entryReferences (matched by entry id +// via `entityByEntryId`), NOT from the aspect's table names: the id is unique +// per entity (two entities can share a table) and survives the +// project-number/id normalization lookupEntry applies to entries but +// lookupEntryLinks does not apply to link references. The aspect then supplies +// the join columns and the direction -- its `source` side is the foreign-key +// side, named by its data source -- used only to orient the two endpoints. The +// link id encodes the (normalized) relationship name. Links are deduped -- +// schema-join is undirected, so a per-entry lookup can return the same link +// once from each endpoint. function readRelationships( - links: EntryLink[], modelName: string, entityEntryNames: Set, - dataSourceToEntity: Map, + links: EntryLink[], modelName: string, entityByEntryId: Map, warnings: string[]): Relationship[] { const prefix = linkNamePrefix(modelName); const seen = new Set(); @@ -339,14 +342,18 @@ function readRelationships( for (const link of links) { if (!link.entryLinkType?.endsWith('/entryLinkTypes/schema-join')) continue; const refs = link.entryReferences ?? []; - // Only a link whose BOTH endpoints are this model's entities belongs here. - if (refs.length !== 2 || !refs.every(r => entityEntryNames.has(r.name))) { - continue; - } - if (link.name) { - if (seen.has(link.name)) continue; - seen.add(link.name); - } + if (refs.length !== 2) continue; + // Resolve both endpoints from the link's references. A reference that is + // not one of this model's entities (another model, or a non-entity) means + // the link does not belong here -- skip it quietly, as with M:N edges. + const endA = entityByEntryId.get(idOf(refs[0].name)); + const endB = entityByEntryId.get(idOf(refs[1].name)); + if (!endA || !endB) continue; + + const key = linkDedupKey(link); + if (seen.has(key)) continue; + seen.add(key); + const join = asArray(aspectDataOf(link.aspects, 'schema-join').joins)[0]; if (!join) { warnings.push( @@ -354,19 +361,35 @@ function readRelationships( `relationship is skipped`); continue; } - const source = dataSourceToEntity.get((join.source?.name ?? '').trim()); - const destination = - dataSourceToEntity.get((join.target?.name ?? '').trim()); - if (!source || !destination) { + // Direction lives in the aspect: `source` is the foreign-key side, named by + // its data source. Orient the two endpoints by matching that name, keeping + // join.source.fields paired with the source side. When neither orientation + // matches -- or both do, because the endpoints share a data source -- the + // direction is undecidable, so keep the reference order and warn rather + // than drop the edge. + const srcName = (join.source?.name ?? '').trim(); + const tgtName = (join.target?.name ?? '').trim(); + const aSrc = (endA.dataSource ?? '').trim(); + const bSrc = (endB.dataSource ?? '').trim(); + const endAIsSource = aSrc === srcName && bSrc === tgtName; + const endBIsSource = bSrc === srcName && aSrc === tgtName; + let [source, destination] = [endA, endB]; + if (endBIsSource && !endAIsSource) { + [source, destination] = [endB, endA]; + } else if (!endAIsSource && !endBIsSource) { warnings.push( - `entry link '${link.name}': a join endpoint table does not match a ` + - `reconstructed entity; the relationship is skipped`); - continue; + `entry link '${link.name}': join direction does not match either ` + + `endpoint's data source; using the reference order`); + } else if (endAIsSource && endBIsSource) { + warnings.push( + `entry link '${link.name}': join direction is ambiguous (endpoints ` + + `share a data source); using the reference order`); } const rel: Relationship = { name: relationshipName(link.name, prefix), - source: {entity: source, columns: asArray(join.source?.fields)}, - destination: {entity: destination, columns: asArray(join.target?.fields)}, + source: {entity: source.name, columns: asArray(join.source?.fields)}, + destination: + {entity: destination.name, columns: asArray(join.target?.fields)}, }; if (join.description !== undefined) rel.description = join.description; out.push(rel); @@ -375,6 +398,16 @@ function readRelationships( } +// A stable dedup key for an entry link: its name when present, else its +// (order-independent) endpoint pair and type. Guards against a per-endpoint +// lookup returning an unnamed link once from each of its two endpoints. +export function linkDedupKey(link: EntryLink): string { + if (link.name) return link.name; + const refs = (link.entryReferences ?? []).map(r => r.name).sort(); + return `${refs.join('')}${link.entryLinkType ?? ''}`; +} + + // Best-effort recovery of the relationship name from the link id, which the // emitter built as linkSlug(`-`) (lowercased/hyphenated -- see // knowledge_catalog.ts). Strips the model prefix when present; the name comes diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index 24d338c3..a1ebae76 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -19,7 +19,7 @@ import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; import {SemanticModel} from './ir'; -import {idOf, modelsFromCatalogResources} from './kc_converter'; +import {idOf, linkDedupKey, modelsFromCatalogResources} from './kc_converter'; export interface KcPullOptions { project: string; @@ -96,7 +96,7 @@ export async function pullKnowledgeCatalog( // Second fetch pass: relationships are schema-join entry links, which the // entry list/lookup does not return. The catalog exposes links only per // referenced entry (:lookupEntryLinks), so fan out over the entity entries - // and dedup by link name -- schema-join is undirected, so each link comes + // and dedup (linkDedupKey) -- schema-join is undirected, so each link comes // back once from each of its two endpoints. const entityEntries = hydrated.filter( e => e.entryType?.endsWith('/entryTypes/semantic-entity')); @@ -124,9 +124,9 @@ export async function pullKnowledgeCatalog( continue; } for (const link of r.links ?? []) { - const key = link.name ?? ''; - if (key && seenLinks.has(key)) continue; - if (key) seenLinks.add(key); + const key = linkDedupKey(link); + if (seenLinks.has(key)) continue; + seenLinks.add(key); entryLinks.push(link); } } diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 0e2f0eb6..88a58a22 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -156,6 +156,129 @@ describe('relationship recovery (schema-join links -> IR)', () => { // The emitter never publishes M:N, so no schema-join link exists to read. expect(roundTrip(model).models[0].relationships).toEqual([]); }); + + test( + 'endpoints resolve from the link references, not the shared table name', + () => { + // Two entities over the SAME table: a data-source index would collapse + // them (last write wins), so both endpoints must come from the link's + // entryReferences instead. Direction can't be told from the shared + // table, so the reader keeps the reference order and warns. + const sameTable: Entity[] = [ + { + name: 'parent_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'id', expression: 'parent_order.id'}], + }, + { + name: 'child_order', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'parent_id', expression: 'child_order.parent_id'}], + }, + ]; + const model: SemanticModel = { + name: 'sales', + entities: sameTable, + relationships: [{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }], + metrics: [], + }; + const {models, warnings} = roundTrip(model); + expect(models[0].relationships).toEqual([{ + name: 'parents', + source: {entity: 'child_order', columns: ['parent_id']}, + destination: {entity: 'parent_order', columns: ['id']}, + }]); + expect(warnings.some(w => /join direction is ambiguous/i.test(w))) + .toBe(true); + }); + + test( + 'endpoints resolve when link references carry an un-normalized project ' + + 'number', + () => { + // The live path is asymmetric: lookupEntry normalizes entry names to + // project IDs, but lookupEntryLinks returns references still carrying + // the numeric project. Matching on the (stable) entry id keeps the + // relationship resolvable rather than silently dropping it. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const numeric = entryLinks.map( + l => ({ + ...l, + entryReferences: + (l.entryReferences ?? + []).map(r => ({ + ...r, + name: r.name.replace( + 'projects/dest/', 'projects/000000000000/'), + })), + })); + const {models} = modelsFromCatalogResources(entries, numeric); + expect(models[0].relationships).toEqual([{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }]); + }); + + test( + 'the model prefix is stripped across tricky model names ' + + '(linkNamePrefix tracks the emitter slug)', + () => { + // Pins the read-side prefix reproduction to the emitter's linkSlug: if + // it drifts, the model prefix would survive and the recovered name + // would not equal the bare, normalized relationship name. + for (const modelName of ['Sales', 'Sales Analytics', '123 Sales!']) { + const model: SemanticModel = { + name: modelName, + entities: twoEntities, + relationships: [{ + name: 'rel_one', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('rel-one'); + } + }); + + test('an unnamed link returned from both endpoints is deduped', () => { + // Defense in depth: if the catalog ever returns a nameless schema-join + // link, the per-endpoint fan-out yields it twice; dedup falls back to the + // endpoint pair so it becomes one relationship, not two. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {models} = modelsFromCatalogResources( + entries, [...nameless, ...nameless.map(l => ({...l}))]); + expect(models[0].relationships).toHaveLength(1); + }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index d5d22d42..2316003d 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -183,6 +183,25 @@ describe('pullKnowledgeCatalog: relationship links', () => { expect(lookupLinks).toHaveBeenCalledTimes(2); }); + test( + 'an unnamed link returned from both endpoints is deduped to one edge', + async () => { + const {entries, entryLinks} = generateCatalogResources(SALES_REL, OPTS); + // A nameless link can't dedup by name; the puller must fall back to the + // endpoint pair, or the per-endpoint fan-out would yield it twice. + const nameless = entryLinks.map(l => ({...l, name: undefined})); + const {lookupLinks} = stubClient(entries, entries, undefined, nameless); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + + // Deduped to a single edge (a nameless link has no id, so the name + // itself can't be recovered -- but it must not be counted twice). + expect(models[0].relationships).toHaveLength(1); + expect(models[0].relationships[0].source.entity).toBe('orders'); + expect(lookupLinks).toHaveBeenCalledTimes(2); + }); + test( 'a failed link lookup on one endpoint still recovers the edge from the ' + 'other, and warns', From e6d5e1f97652082ebf944f2dd3fc8190bccb0e86 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:29:14 +0000 Subject: [PATCH 07/14] mdcode: assert pull symmetry + complete the sales BQ golden Two fixture-coverage gaps from the round-trip review: 1. Symmetry assertion. The golden pull files let a human eyeball what a Knowledge Catalog round trip drops, but nothing asserted it. Add a symmetry test over the converter corpus: load the authored IR, run a full emit -> read round trip, and assert the result equals the authored IR reduced to the "KC floor" (stripToKcFloor) -- the documented losses and normalizations applied to both sides. An undocumented regression (a dropped column, a lost description, an un-stripped M:N edge) now fails here even though each individual loss is already pinned by a targeted test. Export linkNamePrefix so the normalizer reproduces the relationship slug rather than reimplementing it. 2. sales_bq_graph_target had OSI/KC/pull goldens but no BigQuery golden. Add it to the BigQuery corpus and generate the golden: a valid single-node property graph with a MEASURE, so the fixture now carries a complete four-arm round-trip suite. No production behavior change; reader/emitter untouched apart from the linkNamePrefix export. --- .../mdcode/src/libts/semantic/kc_converter.ts | 6 +- .../tests/libts/semantic/bigquery.e2e.test.ts | 1 + .../sales_bq_graph_target.bigquery.golden.sql | 13 ++ .../tests/libts/semantic/kc_converter.test.ts | 134 +++++++++++++++++- 4 files changed, 150 insertions(+), 4 deletions(-) create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 3600d781..7a04d7b2 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -425,8 +425,10 @@ function relationshipName( // Mirrors the model-name portion of the emitter's linkSlug so the prefix a link // id was built with can be stripped. Kept local rather than imported: this -// read-side module must not depend on the write-side emitter. -function linkNamePrefix(modelName: string): string { +// read-side module must not depend on the write-side emitter. Exported for the +// symmetry test, which reuses it to reproduce the normalized relationship name +// rather than reimplementing the slug rule. +export function linkNamePrefix(modelName: string): string { return modelName.toLowerCase() .replace(/[^a-z0-9-]+/g, '-') .replace(/-+/g, '-') diff --git a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts index b16be403..17816f02 100644 --- a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts @@ -41,6 +41,7 @@ const CORPUS = [ 'measure_lowering.yaml', 'metric_skips.yaml', 'keyless_dimension.yaml', + 'sales_bq_graph_target.yaml', ]; // Loads a fixture file to its IR. Split out from `build` so a test that only diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql new file mode 100644 index 00000000..51059fe9 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql @@ -0,0 +1,13 @@ +CREATE OR REPLACE PROPERTY GRAPH `sqlgen-testing.demo.sales` +NODE TABLES ( + `demo.sales.orders` AS orders + KEY(o_orderkey) + PROPERTIES( + o_orderkey, + o_totalprice, + MEASURE(SUM(o_totalprice)) AS total_revenue + ) +); + +-- warnings -- +-- note: no 'BIGQUERY' dialect for one or more expressions; using the portable 'ANSI_SQL' dialect verbatim ('BIGQUERY' accepts the ANSI core subset — supply 'BIGQUERY' variants only for BIGQUERY-specific SQL) diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 88a58a22..91967af8 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -19,9 +19,10 @@ import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; import * as path from 'node:path'; -import {Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {CustomExtension, Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; +import {linkNamePrefix, modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; +import {loadModels} from '../../../src/libts/semantic/loader'; import {serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -559,3 +560,132 @@ describe( }); } }); + + +// -- Symmetry: emit -> read drops ONLY the documented set. -- +// +// The golden pull above lets a human eyeball what a Knowledge Catalog round +// trip loses; this asserts it mechanically. For each corpus fixture it loads +// the authored IR, runs a full emit -> read round trip, and asserts the result +// equals the authored IR reduced to the "KC floor" -- the model with exactly +// the documented losses and normalizations applied (`stripToKcFloor`). Applying +// that same reduction to BOTH sides makes `toEqual` flag only UNDOCUMENTED +// divergence, so a new emitter/reader regression (a dropped column, a lost +// description, an un-stripped M:N edge) fails here even though every individual +// loss is already pinned by a targeted test above. +describe( + 'symmetry: an emit -> read round trip drops only documented fields', () => { + const CORPUS = [ + 'sales_bq_graph_target.yaml', + 'star_orders_customer.yaml', + 'tpcds_date_edge.yaml', + ]; + // Same load defaults as the OSI / KC / pull goldens. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; + + for (const fixture of CORPUS) { + test(fixture, () => { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + for (const authored of loadModels(text, LOAD).models) { + const {models, warnings} = roundTrip(authored); + expect(models).toHaveLength(1); + expect(stripToKcFloor(models[0])).toEqual(stripToKcFloor(authored)); + // A clean corpus model round-trips without reader warnings. + expect(warnings).toEqual([]); + } + }); + } + }); + + +// Reduces a model to the "KC floor": the most an emit -> read round trip can +// preserve, i.e. the authored model with exactly the documented Knowledge +// Catalog losses and normalizations applied. Idempotent (already-floored input +// is unchanged), so it can be applied to both sides of a round trip. Each +// transform mirrors one emitter/reader behavior pinned by a targeted test +// above. +function stripToKcFloor(model: SemanticModel): SemanticModel { + const m = structuredClone(model); + + // Model ai_context is not persisted; only a GOOGLE deployment-targets block + // rides back, re-serialized to the reader's canonical JSON form. + delete m.aiContext; + const ext = canonicalGoogleExt(m.customExtensions); + if (ext) { + m.customExtensions = ext; + } else { + delete m.customExtensions; + } + + for (const e of m.entities) { + e.keys = []; // keys / unique keys are never persisted + delete e.uniqueKeys; + delete e.aiContext; + delete e.customExtensions; + for (const f of e.fields) { + delete f.label; // display label is not persisted + delete f.aiContext; + delete f.importedExpression; // vendor SQL is not persisted + delete f.importedDialect; + delete f.customExtensions; + if (f.dimension) f.dimension = {}; // only the DIMENSION role survives + // String is indistinguishable from an un-typed field on read. + if (f.type === 'String') delete f.type; + } + } + + for (const metric of m.metrics) { + delete metric.aiContext; + delete metric.importedExpression; + delete metric.importedDialect; + delete metric.customExtensions; + // The emitter writes a required dataType, defaulting typeless -> NUMERIC, + // which reads back as Decimal; String collapses to un-typed like fields. + if (metric.type === undefined) { + metric.type = 'Decimal'; + } else if (metric.type === 'String') { + delete metric.type; + } + } + + // M:N (association) edges are never published; a 1:1 / 1:N name comes back + // normalized via the emitter's slug (its exact form is pinned above). + m.relationships = m.relationships.filter(r => !r.association).map(r => { + const rel = structuredClone(r); + rel.name = linkNamePrefix(rel.name); + delete rel.aiContext; + delete rel.customExtensions; + return rel; + }); + + return m; +} + + +// The inverse-canonical of the emitter's deployment-target persistence, +// matching the reader's readDeploymentTargets: keep only GOOGLE +// deploymentTargets and re-serialize them so an authored block and its +// round-tripped form (which the reader always JSON.stringifies afresh) compare +// equal. Returns undefined when the model declares none, exactly as the reader +// omits custom_extensions then. +function canonicalGoogleExt(exts: CustomExtension[]|undefined): + CustomExtension[]|undefined { + const targets: string[] = []; + for (const ext of exts ?? []) { + if (ext.vendorName !== 'GOOGLE') continue; + let parsed: any; + try { + parsed = JSON.parse(ext.data); + } catch { + continue; + } + for (const t of parsed?.deploymentTargets ?? []) { + if (typeof t === 'string' && t) targets.push(t); + } + } + return targets.length ? [{ + vendorName: 'GOOGLE', + data: JSON.stringify({deploymentTargets: targets}) + }] : + undefined; +} From efcc162eeb0c12aa0fb0fccc35785857954665c3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 07:40:48 +0000 Subject: [PATCH 08/14] mdcode: drop the sales_bq_graph_target BigQuery golden The BigQuery corpus golden path names the graph from the test's build opts (sqlgen-testing.demo.sales), ignoring the fixture's deployment target -- so the golden neither reflected the fixture's purpose nor matched a real deploy (demo.sales.sales_graph). That target-driven name is already covered by deploy_bigquery.test.ts, making this golden redundant. Revert the corpus addition and remove the generated file; the fixture keeps its OSI/KC/pull goldens and the pull symmetry assertion. --- .../tests/libts/semantic/bigquery.e2e.test.ts | 1 - .../sales_bq_graph_target.bigquery.golden.sql | 13 ------------- 2 files changed, 14 deletions(-) delete mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql diff --git a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts index 17816f02..b16be403 100644 --- a/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/bigquery.e2e.test.ts @@ -41,7 +41,6 @@ const CORPUS = [ 'measure_lowering.yaml', 'metric_skips.yaml', 'keyless_dimension.yaml', - 'sales_bq_graph_target.yaml', ]; // Loads a fixture file to its IR. Split out from `build` so a test that only diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql deleted file mode 100644 index 51059fe9..00000000 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.bigquery.golden.sql +++ /dev/null @@ -1,13 +0,0 @@ -CREATE OR REPLACE PROPERTY GRAPH `sqlgen-testing.demo.sales` -NODE TABLES ( - `demo.sales.orders` AS orders - KEY(o_orderkey) - PROPERTIES( - o_orderkey, - o_totalprice, - MEASURE(SUM(o_totalprice)) AS total_revenue - ) -); - --- warnings -- --- note: no 'BIGQUERY' dialect for one or more expressions; using the portable 'ANSI_SQL' dialect verbatim ('BIGQUERY' accepts the ANSI core subset — supply 'BIGQUERY' variants only for BIGQUERY-specific SQL) From ff04848e30c91b875d6dfcf0c138639f36162ce7 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Sun, 9 Aug 2026 15:41:56 +0000 Subject: [PATCH 09/14] mdcode: fix pull reader review findings Address code-review findings on the KC->IR reader (kc_converter.ts): - readMetric: when the expression does not pin exactly one known entity (none, or several), fall back to the attach entity metricAspectData persisted instead of dropping it. A cross-entity metric now recovers its authored entity. - readField: skip a schema field with no name (warn) instead of emitting a Field with an undefined name into the entity. - linkNamePrefix: mirror linkSlug's 63-char cap and trailing-hyphen re-strip so the read-side prefix stays aligned with the emitter's link id. - Header comment: add importedExpression (vendor SQL) and String/Opaque- typed metrics to the documented round-trip loss list. Add regression tests for the metric-entity fallback and the nameless-field skip. --- .../mdcode/src/libts/semantic/kc_converter.ts | 48 +++++++++++---- .../tests/libts/semantic/kc_converter.test.ts | 60 +++++++++++++++++++ 2 files changed, 98 insertions(+), 10 deletions(-) diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 7a04d7b2..e732ad55 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -28,14 +28,18 @@ // inverse of the WRITE, not of the authored document. It recovers names, // descriptions, data sources, field datatypes (via the schema aspect) and // DIMENSION roles, field/metric expressions, each metric's attach entity -// (re-derived from its expression, as the loader does), the model's deployment -// targets (from the semantic-model aspect, back into the GOOGLE -// `custom_extensions` block), and 1:1 / 1:N relationships (from the -// `schema-join` entry links a pull fetched -- see +// (re-derived from its expression, as the loader does, else the value the +// emitter persisted), the model's deployment targets (from the semantic-model +// aspect, back into the GOOGLE `custom_extensions` block), and 1:1 / 1:N +// relationships (from the `schema-join` entry links a pull fetched -- see // `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what // the emitter does not write: entity keys/unique keys, `ai_context`, field -// labels, `importedDialect`, and many-to-many (association) relationships -// (whose edge lives only in the BigQuery property graph). Relationship NAMES +// labels, `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// many-to-many (association) relationships (whose edge lives only in the +// BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads +// back un-typed: the metric aspect persists only `dataType` (both collapse to +// `STRING`, with no metadataType to disambiguate), so -- as with a plain STRING +// field -- the reader leaves it un-typed rather than guess. Relationship NAMES // come back normalized (lowercased/hyphenated), since the emitter encodes the // name only in the link id (via `linkSlug`), not in the join aspect. @@ -158,7 +162,9 @@ function readEntity(entry: Entry, warnings: string[]): Entity { name, dataSource, keys: [], // not persisted by the emitter; unrecoverable on read - fields: asArray(schema.fields).map(fd => readField(fd, name, warnings)), + fields: asArray(schema.fields) + .map(fd => readField(fd, name, warnings)) + .filter((f): f is Field => f !== undefined), }; const description = entry.entrySource?.description; if (description !== undefined) entity.description = description; @@ -170,12 +176,14 @@ function readEntity(entry: Entry, warnings: string[]): Entity { // schemaAspectData: the datatype from dataType/metadataType, expressions from // the nested `semantics` block, and the DIMENSION role back to a dimension // marker. -function readField(fd: any, entityName: string, warnings: string[]): Field { +function readField(fd: any, entityName: string, warnings: string[]): Field| + undefined { const name = fd?.name; if (name === undefined || name === '') { warnings.push( `entity '${entityName}': a schema field is missing its name; the ` + - `field may not load`); + `field is skipped`); + return undefined; } const field: Field = {name}; const sem = fd?.semantics ?? {}; @@ -203,8 +211,18 @@ function readMetric( if (data.expression !== undefined) metric.expression = data.expression; const exprForRefs = data.expression ?? ''; const referenced = referencedEntityNames(exprForRefs, entityNames); + const persistedEntity = + typeof data.entity === 'string' && data.entity !== '' ? data.entity : + undefined; if (referenced.length === 1) { + // The expression pins exactly one known entity; re-derive it (as the loader + // does) so the attach entity stays consistent with the reconstructed set. metric.entity = referenced[0]; + } else if (persistedEntity !== undefined) { + // The expression does not pin a single known entity (none, or several), so + // fall back to the attach entity the emitter persisted (metricAspectData) + // rather than dropping it. + metric.entity = persistedEntity; } else if (exprForRefs && !referenced.length) { // Parity with the loader's convertMetric: an expression that qualifies no // known entity is flagged as potentially unplaceable downstream. @@ -428,12 +446,22 @@ function relationshipName( // read-side module must not depend on the write-side emitter. Exported for the // symmetry test, which reuses it to reproduce the normalized relationship name // rather than reimplementing the slug rule. +// +// It reproduces linkSlug's normalization AND its 63-char cap + trailing-hyphen +// re-strip, so the prefix stays aligned with the id even when the combined +// `-` slug was truncated. (linkSlug's `|| 'link'` empty-slug +// fallback is intentionally omitted: an empty prefix strips nothing and +// relationshipName then returns the id verbatim, which is already correct. When +// the cap truncates into the model slug itself the relationship name is +// unrecoverable regardless -- relationshipName returns the truncated id.) export function linkNamePrefix(modelName: string): string { return modelName.toLowerCase() .replace(/[^a-z0-9-]+/g, '-') .replace(/-+/g, '-') .replace(/^[^a-z]+/, '') - .replace(/^-+|-+$/g, ''); + .replace(/^-+|-+$/g, '') + .slice(0, 63) + .replace(/-+$/, ''); } diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 91967af8..e63d6bf0 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -392,6 +392,35 @@ describe('field mapping details', () => { expect(back.importedExpression).toBeUndefined(); expect(back.importedDialect).toBeUndefined(); }); + + test('a schema field missing its name is skipped with a warning', () => { + const model: SemanticModel = { + name: 'm', + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: [], + fields: [{name: 'f', expression: 'e.f'}], + }], + relationships: [], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + // Inject a nameless field record into the entity's schema aspect, as a + // malformed catalog could return; the reader must drop it, not emit a field + // with an undefined name. + for (const entry of entries) { + for (const aspect of Object.values(entry.aspects ?? {})) { + const fields = (aspect as {data?: {fields?: unknown[]}}).data?.fields; + if (Array.isArray(fields)) fields.push({dataType: 'INT64'}); + } + } + const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); + expect(models[0].entities[0].fields.map(f => f.name)).toEqual(['f']); + expect( + warnings.some(w => /missing its name; the field is skipped/i.test(w))) + .toBe(true); + }); }); @@ -452,6 +481,37 @@ describe('metric attach entity is re-derived from the expression', () => { expect(byName.get('revenue')!.entity).toBe('orders'); expect(byName.get('mix')!.entity).toBeUndefined(); }); + + test( + 'a cross-entity metric falls back to the persisted attach entity', () => { + const model: SemanticModel = { + name: 'm', + entities: [ + { + name: 'orders', + dataSource: 'p.d.orders', + keys: [], + fields: [{name: 'amt', expression: 'orders.amt'}] + }, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: [], + fields: [{name: 'region', expression: 'customer.region'}] + }, + ], + relationships: [], + // Two entities in the expression, so it cannot be re-derived; the + // authored attach entity must ride back on the persisted aspect. + metrics: [{ + name: 'mix', + expression: 'SUM(orders.amt) / COUNT(customer.region)', + entity: 'orders', + }], + }; + const {models} = roundTrip(model); + expect(models[0].metrics[0].entity).toBe('orders'); + }); }); From 8768a40dda29ee339591ec628568bb4f1db112d3 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 01:33:02 +0000 Subject: [PATCH 10/14] mdcode: reconcile pull with the merged --emit-expressions gating PR #278 landed on main gating the KC SQL-expression fields off by default: the emitter now omits the per-field `semantics` block (field expression + role) and the metric expression unless `--emit-expressions` is set, and the committed emitter golden was regenerated without them. The pull leg was built against the old always-emit behavior, so after rebasing onto main it read back less than its tests and goldens assumed. Reconcile the reader to the new default: - readMetric no longer warns when a metric aspect has no expression -- that is now the expected default, not a malformed aspect. It still derives the attach entity from the expression when one is present, else from the persisted `entity`, and still warns on an expression that pins no known entity. - Reader header + user guide: field/metric expressions and the DIMENSION role are recovered only from an `--emit-expressions` push; a default push -> pull drops them. - Tests: split the round trip into roundTrip (default, drops the semantics block) and roundTripFull (`--emit-expressions`, keeps it); point the lossless-slice, DIMENSION, canonical-expression, and expression-derivation cases at the full round trip, and add a case pinning the default drop. The symmetry floor now strips field/metric expressions and the dimension role. - Regenerate the .pull.golden.yaml fixtures: they drop only expressions and bare dimension markers, matching the default emitter golden. --- toolbox/mdcode/docs/semantic-model.md | 15 ++- .../mdcode/src/libts/semantic/kc_converter.ts | 32 ++++--- .../sales_bq_graph_target.pull.golden.yaml | 12 --- .../star_orders_customer.pull.golden.yaml | 33 ------- .../fixtures/tpcds_date_edge.pull.golden.yaml | 94 ------------------- .../tests/libts/semantic/kc_converter.test.ts | 78 +++++++++++---- .../tests/libts/semantic/pull_kc.test.ts | 8 +- 7 files changed, 95 insertions(+), 177 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b3a1fcbb..b4aaa106 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -266,17 +266,24 @@ with no matching catalog entry) is left untouched — pull never deletes. > > **Recovered exactly** — these come back as authored: > - Model structure: the model, its entities, and each entity's fields. -> - Field data source and data type; the field expression. -> - Metrics: name, expression, data type, and attach entity. +> - Field data source and data type. +> - Metrics: name, data type, and attach entity. > - 1:1 / 1:N relationships: endpoints, foreign-key direction, and join columns > (from the `schema-join` links). > - Deployment targets. > +> **Recovered only if pushed with `--emit-expressions`** — the per-field +> `semantics` block (expressions and the dimension role) and the metric +> expression are omitted from the catalog by default (see the note above), so +> pull returns them only when the push that wrote them used `--emit-expressions`: +> - Field expressions and metric expressions (the canonical GoogleSQL/ANSI form). +> - A field's dimension role, which comes back as a bare `dimension: {}` marker, +> without its detail (`is_time`, and so on). A default push drops the marker +> entirely. +> > **Recovered, but normalized** — the content survives, the form changes: > - Relationship *names* come back lowercased/hyphenated (the catalog stores the > name only in the link id, e.g. `Places Order` → `places-order`). -> - A field marked as a dimension comes back as a bare `dimension: {}` marker, -> without its detail (`is_time`, and so on). > - A metric authored with no data type comes back as an explicit `Decimal` > (push must write a type, and defaults it to `NUMERIC`). > diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index e732ad55..9c863290 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -26,15 +26,20 @@ // // Fidelity is bounded by what the emitter persisted, so this read is the // inverse of the WRITE, not of the authored document. It recovers names, -// descriptions, data sources, field datatypes (via the schema aspect) and -// DIMENSION roles, field/metric expressions, each metric's attach entity -// (re-derived from its expression, as the loader does, else the value the -// emitter persisted), the model's deployment targets (from the semantic-model -// aspect, back into the GOOGLE `custom_extensions` block), and 1:1 / 1:N -// relationships (from the `schema-join` entry links a pull fetched -- see -// `modelsFromCatalogResources`'s `entryLinks` argument). It cannot recover what -// the emitter does not write: entity keys/unique keys, `ai_context`, field -// labels, `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// descriptions, data sources, field datatypes (via the schema aspect), each +// metric's attach entity (re-derived from its expression, as the loader does, +// when the catalog holds one, else the value the emitter persisted), the +// model's deployment targets (from the semantic-model aspect, back into the +// GOOGLE `custom_extensions` block), and 1:1 / 1:N relationships (from the +// `schema-join` entry links a pull fetched -- see +// `modelsFromCatalogResources`'s `entryLinks` argument). The per-field +// `semantics` block -- field/metric expressions and the DIMENSION role -- is +// gated off the catalog by default: the emitter writes it only under +// `--emit-expressions` (see `KcGenerateOptions.emitExpressions`), so those +// three recover only when the push that wrote them enabled it, and a default +// push -> pull drops them. It cannot recover what the emitter never writes: +// entity keys/unique keys, `ai_context`, field labels, +// `importedExpression`/`importedDialect` (the vendor-dialect SQL), and // many-to-many (association) relationships (whose edge lives only in the // BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads // back un-typed: the metric aspect persists only `dataType` (both collapse to @@ -197,15 +202,14 @@ function readField(fd: any, entityName: string, warnings: string[]): Field| // Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` -// is re-derived from the expression (as the loader does) rather than read from -// the aspect, so it stays consistent with the reconstructed entity set. +// is re-derived from the expression (as the loader does) when the aspect holds +// one, else it falls back to the entity the emitter persisted. A default push +// omits the expression entirely (it is gated behind `--emit-expressions`), so +// an absent expression is expected, not an error, and does not warn. function readMetric( entry: Entry, entityNames: string[], warnings: string[]): Metric { const name = entry.entrySource?.displayName ?? idOf(entry.name); const data = aspectData(entry, 'semantic-metric'); - if (data.expression === undefined) { - warnings.push(`metric '${name}': no expression in semantic-metric aspect`); - } const metric: Metric = {name}; if (data.expression !== undefined) metric.expression = data.expression; diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml index 06dd8bdc..f966f619 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -10,19 +10,7 @@ semantic_model: source: demo.sales.orders fields: - name: o_orderkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderkey - name: o_totalprice - expression: - dialects: - - dialect: BIGQUERY - expression: o_totalprice metrics: - name: total_revenue - expression: - dialects: - - dialect: BIGQUERY - expression: SUM(orders.o_totalprice) datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml index d7a9cbe1..ceb6bed1 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -12,40 +12,15 @@ semantic_model: description: One row per order fields: - name: o_orderkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderkey description: Order identifier - name: o_custkey - expression: - dialects: - - dialect: BIGQUERY - expression: o_custkey - name: o_orderdate - expression: - dialects: - - dialect: BIGQUERY - expression: o_orderdate - dimension: {} - name: o_totalprice - expression: - dialects: - - dialect: BIGQUERY - expression: o_totalprice - name: customer source: samples.tpch.customer fields: - name: c_custkey - expression: - dialects: - - dialect: BIGQUERY - expression: c_custkey - name: c_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_name description: Customer name relationships: - name: orders-to-customer @@ -57,16 +32,8 @@ semantic_model: - c_custkey metrics: - name: total_revenue - expression: - dialects: - - dialect: BIGQUERY - expression: SUM(orders.o_totalprice) datatype: Decimal description: Total order revenue - name: order_count - expression: - dialects: - - dialect: BIGQUERY - expression: COUNT(orders.o_orderkey) datatype: Decimal description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml index 1900fe12..ab45bace 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -9,138 +9,44 @@ semantic_model: description: Fact table containing all store sales transactions fields: - name: ss_item_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_item_sk - dimension: {} - name: ss_ticket_number - expression: - dialects: - - dialect: BIGQUERY - expression: ss_ticket_number - dimension: {} - name: ss_customer_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_customer_sk - dimension: {} - name: ss_store_sk - expression: - dialects: - - dialect: BIGQUERY - expression: ss_store_sk - dimension: {} - name: ss_quantity - expression: - dialects: - - dialect: BIGQUERY - expression: ss_quantity description: Quantity of items sold - name: ss_sales_price - expression: - dialects: - - dialect: BIGQUERY - expression: ss_sales_price description: Sales price per unit - name: ss_ext_sales_price - expression: - dialects: - - dialect: BIGQUERY - expression: ss_ext_sales_price description: Extended sales price (quantity * price) - name: ss_net_profit - expression: - dialects: - - dialect: BIGQUERY - expression: ss_net_profit description: Net profit from the sale - name: customer source: tpcds.public.customer description: Customer dimension with demographic information fields: - name: c_customer_sk - expression: - dialects: - - dialect: BIGQUERY - expression: c_customer_sk - dimension: {} - name: c_first_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_first_name - dimension: {} description: Customer first name - name: c_last_name - expression: - dialects: - - dialect: BIGQUERY - expression: c_last_name - dimension: {} description: Customer last name - name: item source: tpcds.public.item description: Item/Product dimension fields: - name: i_item_sk - expression: - dialects: - - dialect: BIGQUERY - expression: i_item_sk - dimension: {} - name: i_brand - expression: - dialects: - - dialect: BIGQUERY - expression: i_brand - dimension: {} - name: i_category - expression: - dialects: - - dialect: BIGQUERY - expression: i_category - dimension: {} - name: i_current_price - expression: - dialects: - - dialect: BIGQUERY - expression: i_current_price description: Current price of the item - name: store source: tpcds.public.store description: Store dimension with location attributes fields: - name: s_store_sk - expression: - dialects: - - dialect: BIGQUERY - expression: s_store_sk - dimension: {} - name: s_store_name - expression: - dialects: - - dialect: BIGQUERY - expression: s_store_name - dimension: {} - name: s_city - expression: - dialects: - - dialect: BIGQUERY - expression: s_city - dimension: {} - name: s_state - expression: - dialects: - - dialect: BIGQUERY - expression: s_state - dimension: {} - name: s_number_employees - expression: - dialects: - - dialect: BIGQUERY - expression: s_number_employees description: Number of employees at the store - name: date_dim source: sqlgen-testing.demo.date_dim diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index e63d6bf0..4ccac618 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -7,13 +7,16 @@ // emitter is lossless. The write drops content by design (entity keys, // ai_context, field labels, importedDialect, and many-to-many relationships -- // see the emitter header), so the expected read-back is the source model with -// exactly those fields cleared. 1:1 / 1:N relationships and deployment targets -// DO round-trip (via schema-join links and the semantic-model aspect), except -// that relationship names come back normalized (lowercased/hyphenated). -// Targeted tests pin the mapping details a round trip cannot isolate (the -// dataType inverse, the DIMENSION role, resource-URI parsing, metric attach -// re-derivation, relationship endpoint/direction recovery, and parent/anchor -// grouping). +// exactly those fields cleared. It ALSO drops the per-field `semantics` block +// (field/metric expressions and the DIMENSION role) unless the push enabled +// `emitExpressions`, so a DEFAULT round trip (roundTrip) loses those too, while +// a full round trip (roundTripFull, emitExpressions: true) keeps them. 1:1 / +// 1:N relationships and deployment targets DO round-trip either way (via +// schema-join links and the semantic-model aspect), except that relationship +// names come back normalized (lowercased/hyphenated). Targeted tests pin the +// mapping details a round trip cannot isolate (the dataType inverse, the +// DIMENSION role, resource-URI parsing, metric attach re-derivation, +// relationship endpoint/direction recovery, and parent/anchor grouping). import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; @@ -33,18 +36,32 @@ const OPTS = { entryGroup: 'eg' }; -// Emits a model to entries + entry links and reads it straight back. +// Emits a model to entries + entry links and reads it straight back. The +// default push omits the per-field `semantics` block (expressions + role), so a +// default round trip drops those; use roundTripFull to exercise their recovery. function roundTrip(model: SemanticModel): {models: SemanticModel[]; warnings: string[]} { const {entries, entryLinks} = generateCatalogResources(model, OPTS); return modelsFromCatalogResources(entries, entryLinks); } +// A round trip through a `--emit-expressions` push: the catalog then holds the +// per-field `semantics` block and the metric expression, so field/metric +// expressions and DIMENSION roles round-trip too. +function roundTripFull(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries, entryLinks} = + generateCatalogResources(model, {...OPTS, emitExpressions: true}); + return modelsFromCatalogResources(entries, entryLinks); +} + describe('emitter -> reader round trip (lossless slice)', () => { // A model using only round-trippable content: no keys/ai_context/labels/ // relationships (all dropped by the write), and datatypes that invert - // cleanly. + // cleanly. Its fields and metric carry expressions and a DIMENSION role, so + // it round-trips losslessly only through a `--emit-expressions` push + // (roundTripFull); a default push omits the per-field `semantics` block. const source: SemanticModel = { name: 'sales', description: 'the sales model', @@ -73,7 +90,7 @@ describe('emitter -> reader round trip (lossless slice)', () => { }; test('reconstructs an IR equal to the source', () => { - const {models} = roundTrip(source); + const {models} = roundTripFull(source); expect(models).toHaveLength(1); expect(models[0]).toEqual(source); }); @@ -357,30 +374,46 @@ describe('dataType inverse (schema aspect -> IR type)', () => { describe('field mapping details', () => { - function readField(field: Entity['fields'][number]) { + function readWith( + rt: (m: SemanticModel) => {models: SemanticModel[]}, + field: Entity['fields'][number]) { const model: SemanticModel = { name: 'm', entities: [{name: 'e', dataSource: 'p.d.t', keys: [], fields: [field]}], relationships: [], metrics: [], }; - return roundTrip(model).models[0].entities[0].fields[0]; + return rt(model).models[0].entities[0].fields[0]; } + // Default push (no per-field `semantics`) vs a `--emit-expressions` push. + const readField = (f: Entity['fields'][number]) => readWith(roundTrip, f); + const readFieldFull = (f: Entity['fields'][number]) => + readWith(roundTripFull, f); test('a DIMENSION role reads back as a dimension marker', () => { - const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + // The role lives in the gated `semantics` block, so it survives only a + // `--emit-expressions` push. + const back = readFieldFull({name: 'd', expression: 'e.d', dimension: {}}); expect(back.dimension).toEqual({}); }); test('a non-dimension field has no dimension marker', () => { - const back = readField({name: 'f', expression: 'e.f'}); + const back = readFieldFull({name: 'f', expression: 'e.f'}); expect(back.dimension).toBeUndefined(); }); + test( + 'a default push drops the DIMENSION role with the semantics block', + () => { + const back = readField({name: 'd', expression: 'e.d', dimension: {}}); + expect(back.dimension).toBeUndefined(); + expect(back.expression).toBeUndefined(); + }); + test( 'an imported expression is not persisted to or recovered from the catalog', () => { - const back = readField({ + const back = readFieldFull({ name: 'amt', expression: 'e.amt', importedExpression: 'e.amt::NUMBER', @@ -476,7 +509,9 @@ describe('metric attach entity is re-derived from the expression', () => { }, ], }; - const {models} = roundTrip(model); + // Full push: the expression is in the catalog, so the entity is + // re-derived from it (single entity -> attach; two entities -> none). + const {models} = roundTripFull(model); const byName = new Map(models[0].metrics.map(m => [m.name, m])); expect(byName.get('revenue')!.entity).toBe('orders'); expect(byName.get('mix')!.entity).toBeUndefined(); @@ -559,7 +594,9 @@ describe('metric expression referencing no known entity', () => { // References `widgets`, which is not an entity of this model. metrics: [{name: 'bogus', expression: 'SUM(widgets.qty)'}], }; - const {models, warnings} = roundTrip(model); + // The unplaceable warning fires from the expression, so it needs a push + // that wrote one (a default push omits it, leaving nothing to check). + const {models, warnings} = roundTripFull(model); expect(models[0].metrics[0].entity).toBeUndefined(); expect(warnings.some(w => /bogus.*references no known entity/i.test(w))) .toBe(true); @@ -688,7 +725,11 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { delete f.importedExpression; // vendor SQL is not persisted delete f.importedDialect; delete f.customExtensions; - if (f.dimension) f.dimension = {}; // only the DIMENSION role survives + // A default push omits the per-field `semantics` block, so the field + // expression and the DIMENSION role are not persisted (they ride back + // only through a `--emit-expressions` push). + delete f.expression; + delete f.dimension; // String is indistinguishable from an un-typed field on read. if (f.type === 'String') delete f.type; } @@ -699,6 +740,7 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { delete metric.importedExpression; delete metric.importedDialect; delete metric.customExtensions; + delete metric.expression; // omitted by a default push, like field ones // The emitter writes a required dataType, defaulting typeless -> NUMERIC, // which reads back as Decimal; String collapses to un-typed like fields. if (metric.type === undefined) { diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index 2316003d..83f5aee0 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -47,9 +47,13 @@ const SALES: SemanticModel = { }], }; -// The entries the emitter would have written for a model. +// The entries the emitter would have written for a model. Emitted with +// expressions on (a `--emit-expressions` push) so a model that carries field / +// metric expressions reconstructs exactly; the default push omits them (the +// converter tests in kc_converter.test.ts pin that gating). function entriesFor(model: SemanticModel): Entry[] { - return generateCatalogResources(model, OPTS).entries; + return generateCatalogResources(model, {...OPTS, emitExpressions: true}) + .entries; } function ok(result?: T): ApiResult { From b36b7585d15a597ce47526383786b44934ef7433 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 02:34:24 +0000 Subject: [PATCH 11/14] mdcode: document pull round-trip writer-side follow-ups Distinguish inherent pull losses from write-side limits: relationship names (not stored in the schema-join aspect) and non-canonical deployment targets (dropped on write) could round-trip faithfully with a writer change. Reader already recovers everything the catalog holds. --- toolbox/mdcode/docs/semantic-model.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b4aaa106..21b3e5f4 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -298,6 +298,22 @@ with no matching catalog entry) is left untouched — pull never deletes. > pulled document as a faithful copy of the catalog metadata, not of the authored > model, and keep the authored document as the source of truth. +> **Note — writer-side follow-ups (not inherent to pull).** Two of the reductions +> above are limits of what push currently *writes*, not of what pull can recover. +> They are recorded here as write-side follow-ups; the reader (pull) already +> returns everything the catalog holds. +> +> - **Relationship names.** The `schema-join` aspect does not store the authored +> relationship name, so pull recovers it from the link id — which is lowercased +> and hyphenated. Persisting the name in the aspect on write would let pull +> return it verbatim. +> - **Non-canonical deployment targets.** Push persists a target only when it is a +> canonical BigQuery Graph URL +> (`//bigquery.googleapis.com/projects/.../datasets/.../propertyGraphs/...`). +> Other forms — a misspelled path, or the `projects/.../entryGroups/@bigquery/` +> entry form — are dropped on write, so pull has nothing to recover. Widening or +> normalizing the writer's accepted forms would let them round-trip. + ## Permissions `push` needs access to whichever destinations you deploy to. From abc370b1f29f25743be3c9b4663b1276ff7d96b6 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 03:48:54 +0000 Subject: [PATCH 12/14] mdcode: document malformed deployment-target rejection A non-canonical BigQuery Graph deployment target fails push at the validation gate, before any leg and for every --target, so nothing is written to BigQuery or Knowledge Catalog. Correct the earlier writer-side follow-up note, which wrongly implied such targets are silently dropped on write, and sharpen the relationship-name follow-up to name the server-side schema-join aspect-type template as the fix. --- toolbox/mdcode/docs/semantic-model.md | 41 +++++++++++++++++---------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index 21b3e5f4..b111776f 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -180,7 +180,16 @@ paired columns and foreign-key direction — in its aspect. `push` and `--validate-only` run the same checks, **before either destination is touched**, so a model that cannot deploy fails fast instead of half-deploying: -* **Exactly one deployment target per model.** *(static)* +* **Exactly one deployment target per model, and it must be a valid BigQuery + Graph URI.** A model with no target — or with more than one — is rejected, and + so is a single target whose URI does not match + `//bigquery.googleapis.com/projects/

/datasets//propertyGraphs/` (for + example a `propertyGraph`/`propertyGraphs` typo, or a + `…/entryGroups/@bigquery/entries/…` entry form). The error names the offending + URI and the expected form. This gate runs before any destination leg and for + every `--target`, so a malformed target writes **nothing** — not to BigQuery + and **not to Knowledge Catalog**; the push aborts with a non-zero exit and no + entries are created. *(static)* * **Every metric on a BigQuery Graph model resolves to exactly one entity** — otherwise it would be dropped from the BigQuery Graph. Set the metric's attach entity, or scope its expression to a single entity. *(static)* @@ -298,21 +307,23 @@ with no matching catalog entry) is left untouched — pull never deletes. > pulled document as a faithful copy of the catalog metadata, not of the authored > model, and keep the authored document as the source of truth. -> **Note — writer-side follow-ups (not inherent to pull).** Two of the reductions -> above are limits of what push currently *writes*, not of what pull can recover. -> They are recorded here as write-side follow-ups; the reader (pull) already -> returns everything the catalog holds. +> **Note — writer-side follow-up (not inherent to pull).** One reduction above is +> a limit of what push currently *writes*, not of what pull can recover. It is +> recorded here as a write-side follow-up; the reader (pull) already returns +> everything the catalog holds. > -> - **Relationship names.** The `schema-join` aspect does not store the authored -> relationship name, so pull recovers it from the link id — which is lowercased -> and hyphenated. Persisting the name in the aspect on write would let pull -> return it verbatim. -> - **Non-canonical deployment targets.** Push persists a target only when it is a -> canonical BigQuery Graph URL -> (`//bigquery.googleapis.com/projects/.../datasets/.../propertyGraphs/...`). -> Other forms — a misspelled path, or the `projects/.../entryGroups/@bigquery/` -> entry form — are dropped on write, so pull has nothing to recover. Widening or -> normalizing the writer's accepted forms would let them round-trip. +> - **Relationship names.** The `schema-join` aspect type's `metadataTemplate` has +> no field for the relationship name, so push cannot store it and pull recovers +> it from the link id — which is lowercased and hyphenated (the entry-link id +> format forbids the original casing/underscores). Returning the name verbatim +> requires adding a name field to the built-in `schema-join` aspect type in +> Knowledge Catalog (server-side), after which the client write/read is trivial; +> it is the same class of gap as the `semantics` field that gates +> `--emit-expressions`. +> +> (A non-canonical deployment target is **not** a pull gap: push rejects it at the +> validation gate before any leg runs, so it is never written — see +> [Validation](#validation).) ## Permissions From b79a053a8a3756e3dea1fede52cfea96909c110d Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 04:59:42 +0000 Subject: [PATCH 13/14] mdcode: store SQL expressions in the sql-expressions companion aspect Push and pull now move field/metric SQL through a single sql-expressions companion aspect (data_classification METADATA_AND_DATA), per the V2 "SQL Expression Storage in USL Semantic Models" proposal, instead of the core schema/semantic-metric aspects. The core aspects stay metadata-only; the schema field carries only a DIMENSION marker. A shared codec (sql_expressions.ts) fixes the aspect shape and the qualifier convention -- omitted qualifier = primary GoogleSQL (IR expression), `imported` = the vendor form (IR importedExpression) -- and the emitter and reader are exact inverses over it, so a --emit-expressions push round-trips both expression forms through pull. The source dialect label is still not stored (inferred from importedSystem), so importedDialect stays unset. Both the companion aspect and the DIMENSION marker remain gated behind --emit-expressions (off by default); a default push emits neither and matches the live published system types byte-for-byte. --- toolbox/mdcode/docs/semantic-model.md | 40 ++-- .../semantic/deploy_knowledge_catalog.ts | 8 +- .../mdcode/src/libts/semantic/kc_converter.ts | 88 +++++--- .../src/libts/semantic/knowledge_catalog.ts | 125 +++++++---- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 14 +- .../src/libts/semantic/sql_expressions.ts | 103 +++++++++ toolbox/mdcode/src/tool/commands.ts | 9 +- toolbox/mdcode/src/tool/main.ts | 2 +- .../tests/libts/semantic/kc_converter.test.ts | 66 +++--- .../libts/semantic/knowledge_catalog.test.ts | 209 +++++++++++++----- .../tests/libts/semantic/pull_kc.test.ts | 22 +- .../libts/semantic/sql_expressions.test.ts | 79 +++++++ 12 files changed, 577 insertions(+), 188 deletions(-) create mode 100644 toolbox/mdcode/src/libts/semantic/sql_expressions.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/sql_expressions.test.ts diff --git a/toolbox/mdcode/docs/semantic-model.md b/toolbox/mdcode/docs/semantic-model.md index b111776f..5e3955f4 100644 --- a/toolbox/mdcode/docs/semantic-model.md +++ b/toolbox/mdcode/docs/semantic-model.md @@ -115,7 +115,7 @@ kcmd push --print # also print the generated DDL / entry plan | `--validate-only` | Run every validation check and report pass/fail, but write nothing. | | `--print` | Print each destination's generated artifact (BigQuery DDL, Knowledge Catalog entry plan). Combine with `--validate-only` to preview without deploying. | | `--force-remove` | Delete models in the entry group that this push no longer includes (see [Updating and removing models](#updating-and-removing-models)). | -| `--emit-expressions` | Also write the SQL-expression fields (per-field `schema.semantics` and `semantic-metric.expression`) to Knowledge Catalog. Off by default: the published system-type templates do not carry them yet. Knowledge Catalog push only. | +| `--emit-expressions` | Also write the SQL expressions to Knowledge Catalog: a `sql-expressions` companion aspect (the field/metric expressions, both the primary GoogleSQL form and the imported vendor form) plus the schema field's `semantic` DIMENSION marker. Off by default: the published system-type templates do not carry them yet. Knowledge Catalog push only. | Destinations always deploy BigQuery-first and fail fast, so a rejected model never half-deploys. @@ -164,16 +164,18 @@ paired columns and foreign-key direction — in its aspect. > **Note — push to Knowledge Catalog is lossy.** The catalog holds metadata, > not a full copy of your model. It **stores** names, descriptions, data > sources, field datatypes and roles, and 1:1 / 1:N relationships (as -> `schema-join` links). By default it does **not** store the SQL expressions: -> the published system-type templates do not yet carry a per-field `semantics` -> block or a `semantic-metric.expression` field, so the default push omits them -> (pass `--emit-expressions` to write the canonical GoogleSQL/ANSI expression -> once the templates gain the fields). It never stores entity keys, `ai_context`, -> field labels, the original vendor SQL (`importedExpression` — e.g. the MAQL or -> Snowflake form a metric was imported from), or M:N relationships. Those stay in -> your authored document (and, for the edges, in the BigQuery property graph); the -> vendor SQL and expressions are still used when generating BigQuery SQL. Keep -> your model document as the source of truth. +> `schema-join` links). By default it does **not** store the SQL expressions: the +> core `schema` / `semantic-metric` aspects stay metadata-only, and the +> executable SQL lives in a separate `sql-expressions` companion aspect the +> published system-type templates do not carry yet, so the default push omits it. +> Pass `--emit-expressions` (once that aspect type is provisioned) to write the +> companion aspect — it stores **both** the canonical GoogleSQL/ANSI expression +> and the original vendor form (`importedExpression` — e.g. the MAQL or Snowflake +> form a metric was imported from), under an `imported` qualifier — plus the +> schema field's DIMENSION marker. It never stores entity keys, `ai_context`, +> field labels, the exact imported dialect label, or M:N relationships. Those stay +> in your authored document (and, for the edges, in the BigQuery property graph). +> Keep your model document as the source of truth. ## Validation @@ -281,11 +283,14 @@ with no matching catalog entry) is left untouched — pull never deletes. > (from the `schema-join` links). > - Deployment targets. > -> **Recovered only if pushed with `--emit-expressions`** — the per-field -> `semantics` block (expressions and the dimension role) and the metric -> expression are omitted from the catalog by default (see the note above), so -> pull returns them only when the push that wrote them used `--emit-expressions`: -> - Field expressions and metric expressions (the canonical GoogleSQL/ANSI form). +> **Recovered only if pushed with `--emit-expressions`** — the `sql-expressions` +> companion aspect and the schema DIMENSION marker are omitted from the catalog +> by default (see the note above), so pull returns them only when the push that +> wrote them used `--emit-expressions`: +> - Field expressions and metric expressions — **both** the canonical +> GoogleSQL/ANSI form and the original vendor form (`importedExpression`), the +> latter under the aspect's `imported` qualifier. Only the exact imported +> *dialect label* is not recovered. > - A field's dimension role, which comes back as a bare `dimension: {}` marker, > without its detail (`is_time`, and so on). A default push drops the marker > entirely. @@ -300,7 +305,8 @@ with no matching catalog entry) is left untouched — pull never deletes. > - Entity keys / unique keys. > - `ai_context`. > - Field labels. -> - The original vendor SQL (`importedExpression`). +> - The exact imported dialect label (`importedDialect`) — the vendor SQL itself +> comes back with `--emit-expressions`, but its dialect is not stored. > - M:N relationships (the edge lives only in the BigQuery property graph). > > **So: a push followed by a pull does not return your original file.** Treat a diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index 5676ec86..05b45755 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -57,10 +57,10 @@ export interface KcDeployOptions { systemTypeProject?: string; // Location the built-in system types are referenced from. Default `global`. systemTypeLocation?: string; - // Emit the SQL-expression fields not yet in the published system-type - // templates (per-field `schema.semantics` and `semantic-metric.expression`). - // Off by default so the push matches the live types; see - // KcGenerateOptions.emitExpressions. + // Emit the SQL expressions not yet in the published system-type templates: a + // `sql-expressions` companion aspect on entity/metric entries plus the schema + // field's DIMENSION marker. Off by default so the push matches the live + // types; see KcGenerateOptions.emitExpressions. emitExpressions?: boolean; // Compile and report only; never writes to the catalog (a dry run). validateOnly?: boolean; diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 9c863290..e8c7003f 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -32,14 +32,17 @@ // model's deployment targets (from the semantic-model aspect, back into the // GOOGLE `custom_extensions` block), and 1:1 / 1:N relationships (from the // `schema-join` entry links a pull fetched -- see -// `modelsFromCatalogResources`'s `entryLinks` argument). The per-field -// `semantics` block -- field/metric expressions and the DIMENSION role -- is -// gated off the catalog by default: the emitter writes it only under -// `--emit-expressions` (see `KcGenerateOptions.emitExpressions`), so those -// three recover only when the push that wrote them enabled it, and a default -// push -> pull drops them. It cannot recover what the emitter never writes: -// entity keys/unique keys, `ai_context`, field labels, -// `importedExpression`/`importedDialect` (the vendor-dialect SQL), and +// `modelsFromCatalogResources`'s `entryLinks` argument). Field and metric +// expressions ride in the `sql-expressions` companion aspect and the DIMENSION +// role in the schema field's `semantic` marker; both are gated off the catalog +// by default -- the emitter writes them only under `--emit-expressions` (see +// `KcGenerateOptions.emitExpressions`) -- so they recover only when the push +// that wrote them enabled it, and a default push -> pull drops them. When the +// aspect is present, BOTH the primary GoogleSQL expression AND the imported +// vendor form (`importedExpression`) come back (the imported form under an +// `imported` qualifier); its source dialect is not stored, so `importedDialect` +// stays unset. It cannot recover what the emitter never writes: entity +// keys/unique keys, `ai_context`, field labels, the exact imported dialect, and // many-to-many (association) relationships (whose edge lives only in the // BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads // back un-typed: the metric aspect persists only `dataType` (both collapse to @@ -52,6 +55,7 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {CustomExtension, DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; import {referencedEntityNames} from './sql_expr_utils'; +import {recoverExpressions, SqlExpressionRecord, SqlExpressionsData} from './sql_expressions'; export interface ReadResult { models: SemanticModel[]; @@ -163,12 +167,16 @@ function readEntity(entry: Entry, warnings: string[]): Entity { `entity '${name}': no backing data source in the semantic-entity ` + `aspect; 'source' will be empty and the entity may not load`); } + // Field expressions live in the sql-expressions companion aspect (absent on a + // default push), keyed by field name. Build the lookup once, then join each + // schema field to its records by name. + const exprByField = fieldExpressionRecords(entry); const entity: Entity = { name, dataSource, keys: [], // not persisted by the emitter; unrecoverable on read fields: asArray(schema.fields) - .map(fd => readField(fd, name, warnings)) + .map(fd => readField(fd, exprByField, name, warnings)) .filter((f): f is Field => f !== undefined), }; const description = entry.entrySource?.description; @@ -177,12 +185,14 @@ function readEntity(entry: Entry, warnings: string[]): Entity { } -// Reconstructs a field from one `schema` aspect field record, inverting -// schemaAspectData: the datatype from dataType/metadataType, expressions from -// the nested `semantics` block, and the DIMENSION role back to a dimension -// marker. -function readField(fd: any, entityName: string, warnings: string[]): Field| - undefined { +// Reconstructs a field from one `schema` aspect field record plus its +// expression records from the companion sql-expressions aspect (keyed by name), +// inverting schemaAspectData + entitySqlExpressionsData: the datatype from +// dataType/metadataType, the primary/imported expressions from the companion +// aspect, and the `semantic` DIMENSION marker back to a dimension marker. +function readField( + fd: any, exprByField: Map, + entityName: string, warnings: string[]): Field|undefined { const name = fd?.name; if (name === undefined || name === '') { warnings.push( @@ -191,29 +201,57 @@ function readField(fd: any, entityName: string, warnings: string[]): Field| return undefined; } const field: Field = {name}; - const sem = fd?.semantics ?? {}; - if (sem.expression !== undefined) field.expression = sem.expression; + const {expression, importedExpression} = + recoverExpressions(exprByField.get(name)); + if (expression !== undefined) field.expression = expression; + if (importedExpression !== undefined) { + field.importedExpression = importedExpression; + } const type = irDataType(fd?.dataType, fd?.metadataType); if (type !== undefined) field.type = type; - if (sem.role === 'DIMENSION') field.dimension = {}; + if (fd?.semantic === 'DIMENSION') field.dimension = {}; if (fd?.description !== undefined) field.description = fd.description; return field; } -// Reconstructs a metric from its `semantic-metric` aspect. The attach `entity` -// is re-derived from the expression (as the loader does) when the aspect holds -// one, else it falls back to the entity the emitter persisted. A default push -// omits the expression entirely (it is gated behind `--emit-expressions`), so -// an absent expression is expected, not an error, and does not warn. +// The per-field expression records from an entity's sql-expressions aspect, +// indexed by field name. Empty when a default push wrote no companion aspect. +// Field records missing a name are dropped (they cannot be joined to a schema +// field). +function fieldExpressionRecords(entry: Entry): + Map { + const data = aspectData(entry, 'sql-expressions') as SqlExpressionsData; + const out = new Map(); + for (const f of asArray(data.fields)) { + if (typeof f?.name === 'string' && f.name !== '') { + out.set(f.name, asArray(f.expressions)); + } + } + return out; +} + + +// Reconstructs a metric from its `semantic-metric` aspect (metadata) and its +// `sql-expressions` companion aspect (the aggregate formula). The attach +// `entity` is re-derived from the expression (as the loader does) when the +// catalog holds one, else it falls back to the entity the emitter persisted. A +// default push omits the expression entirely (it is gated behind +// `--emit-expressions`), so an absent expression is expected, not an error, and +// does not warn. function readMetric( entry: Entry, entityNames: string[], warnings: string[]): Metric { const name = entry.entrySource?.displayName ?? idOf(entry.name); const data = aspectData(entry, 'semantic-metric'); + const sql = aspectData(entry, 'sql-expressions') as SqlExpressionsData; const metric: Metric = {name}; - if (data.expression !== undefined) metric.expression = data.expression; - const exprForRefs = data.expression ?? ''; + const {expression, importedExpression} = recoverExpressions(sql.expressions); + if (expression !== undefined) metric.expression = expression; + if (importedExpression !== undefined) { + metric.importedExpression = importedExpression; + } + const exprForRefs = expression ?? ''; const referenced = referencedEntityNames(exprForRefs, entityNames); const persistedEntity = typeof data.entity === 'string' && data.entity !== '' ? data.entity : diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index a3051b22..36c7956c 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -24,12 +24,18 @@ // importedResource? } } // * semantic-metric = { entity?, dataType (required) } // * schema = { fields: [{ name, dataType, metadataType, -// description? }] } +// description?, semantic? }] } // -// The SQL-expression fields -- `semantic-metric.expression` and the per-field -// `schema.semantics` { expression, role } block -- are NOT in the published -// system-type templates yet, so they are gated behind -// KcGenerateOptions.emitExpressions (off by default) and omitted above. +// The core `schema` / `semantic-metric` aspects stay metadata-only: they never +// carry executable SQL. Per the V2 "SQL Expression Storage in USL Semantic +// Models" proposal, SQL lives in a separate `sql-expressions` companion aspect +// (see ./sql_expressions), attached alongside the entity's `schema` and the +// metric's `semantic-metric` aspect and holding the field/metric expressions +// (primary GoogleSQL + the imported vendor form). Both that companion aspect +// AND the schema field's `semantic` (DIMENSION role) marker are gated behind +// KcGenerateOptions.emitExpressions (off by default), because the published +// system-type templates do not carry them yet; a default push emits neither and +// matches the live types byte-for-byte. // // Relationships become `schema-join` entry links between the two entity entries. // schema-join is a built-in, undirected entry link type in `dataplex-types/global` @@ -46,6 +52,7 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {bigQueryGraphTargets} from './deploy_bigquery'; import {DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; +import {expressionRecords, SQL_EXPRESSIONS_ASPECT, SqlExpressionsData} from './sql_expressions'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -59,12 +66,14 @@ export interface KcGenerateOptions { entryGroup: string; // destination entry group systemTypeProject?: string; // default 'dataplex-types' systemTypeLocation?: string; // default 'global' - // Emit the SQL-expression fields the published Dataplex system-type templates - // do not carry yet: the per-field `schema.semantics` block (expression + - // role) and `semantic-metric.expression`. Off by default so a push matches - // the live types; flip on once the templates gain these fields. Enabling it + // Emit the SQL expressions the published Dataplex system-type templates do + // not carry yet: a `sql-expressions` companion aspect (see ./sql_expressions) + // on each entity and metric entry holding the field/metric expressions, plus + // the schema field's `semantic` (DIMENSION role) marker. Off by default so a + // push matches the live types; flip on once the sql-expressions aspect type + // is provisioned and the schema template exposes `semantic`. Enabling it // against today's types fails the push with an ASPECT_*_PARSING_FAILURE on - // the unknown property. + // the unknown aspect/property. emitExpressions?: boolean; } @@ -133,16 +142,23 @@ export function generateCatalogResources( warnings.push( `entity '${entity.name}': no keys declared in the source model`); } + // required_aspects: semantic-entity AND the built-in schema. The + // sql-expressions companion aspect rides alongside only under emitExpr, and + // only when some field actually has an expression (never an empty aspect). + const entityAspects: Record> = { + 'semantic-entity': entityAspectData(entity), + 'schema': schemaAspectData(entity, emitExpr), + }; + if (emitExpr) { + const sql = entitySqlExpressionsData(entity); + if (sql) entityAspects[SQL_EXPRESSIONS_ASPECT] = sql; + } entries.push({ name: names.entry(entityId), entryType: names.typeName('entry', 'semantic-entity'), parentEntry: modelEntryName, entrySource: source(entity.name, entity.description), - // required_aspects: semantic-entity AND the built-in schema. - aspects: aspectMap(names, { - 'semantic-entity': entityAspectData(entity), - 'schema': schemaAspectData(entity, emitExpr), - }), + aspects: aspectMap(names, entityAspects), }); } @@ -150,14 +166,19 @@ export function generateCatalogResources( const metricId = names.metricId(model, metric); if (!claim(seen, metricId, 'entry', `metric '${metric.name}'`, warnings)) continue; + const metricAspects: Record> = { + 'semantic-metric': metricAspectData(metric, warnings), + }; + if (emitExpr) { + const sql = metricSqlExpressionsData(metric); + if (sql) metricAspects[SQL_EXPRESSIONS_ASPECT] = sql; + } entries.push({ name: names.entry(metricId), entryType: names.typeName('entry', 'semantic-metric'), parentEntry: modelEntryName, entrySource: source(metric.name, metric.description), - aspects: aspectMap(names, { - 'semantic-metric': metricAspectData(metric, warnings, emitExpr), - }), + aspects: aspectMap(names, metricAspects), }); } @@ -282,9 +303,13 @@ function entityAspectData(entity: Entity): Record { }; } -// The built-in schema aspect, carrying each field's column type plus the new -// per-field `semantics` block (expression / role). name, -// dataType, and metadataType are required per column. +// The built-in schema aspect: metadata-only (name, dataType, metadataType, +// description), plus the per-field `semantic` DIMENSION marker. Executable SQL +// does NOT live here -- field expressions are emitted in the sql-expressions +// companion aspect (see entitySqlExpressionsData). `semantic` is set only for a +// dimension field and only under emitExpressions (the published `schema` +// template does not carry it yet); a plain field, or any field on a default +// push, leaves it unset -- matching the proposal's "Unset otherwise". function schemaAspectData( entity: Entity, emitExpressions: boolean): Record { return { @@ -294,29 +319,48 @@ function schemaAspectData( dataType: columnDataType(f.type), metadataType: columnMetadataType(f.type), description: f.description, - // The per-field `semantics` block (expression + role) is - // not in the published `schema` aspect template yet, so - // it is gated off by default (see - // KcGenerateOptions.emitExpressions); compact() then - // drops the undefined key. - semantics: emitExpressions ? compact({ - expression: f.expression, - // A field with any dimension metadata is a dimension; - // otherwise DEFAULT. - role: f.dimension ? 'DIMENSION' : 'DEFAULT', - }) : undefined, + semantic: emitExpressions && f.dimension ? 'DIMENSION' : + undefined, })), }; } -// semantic-metric: the model-level aggregate. `dataType` is required by the -// aspect type; when the model does not declare one, fall back to NUMERIC -// (decimal) and warn (metrics are aggregates, so an exact numeric is the -// sensible default; dimensions, in schemaAspectData, default to STRING) rather -// than emit an invalid aspect. +// The entity's sql-expressions aspect: one `fields[]` record per field that +// carries an expression, each with the primary GoogleSQL form (unqualified) and +// the imported vendor form (qualifier `imported`). Fields with no expression +// are omitted; returns undefined when no field has any, so the caller attaches +// the aspect only when it has content. +function entitySqlExpressionsData(entity: Entity): SqlExpressionsData| + undefined { + const fields = (entity.fields ?? []) + .map(f => ({ + name: f.name, + expressions: expressionRecords( + f.expression, f.importedExpression), + })) + .filter(f => f.expressions.length > 0); + return fields.length ? {fields} : undefined; +} + +// The metric's sql-expressions aspect: the aggregate formula as entry-level +// `expressions[]` (primary GoogleSQL + the imported vendor form). Returns +// undefined when the metric has neither expression form. +function metricSqlExpressionsData(metric: Metric): SqlExpressionsData| + undefined { + const expressions = + expressionRecords(metric.expression, metric.importedExpression); + return expressions.length ? {expressions} : undefined; +} + +// semantic-metric: the model-level aggregate. Metadata-only -- the aggregation +// formula lives in the sql-expressions companion aspect (see +// metricSqlExpressionsData), not here. `dataType` is required by the aspect +// type; when the model does not declare one, fall back to NUMERIC (decimal) and +// warn (metrics are aggregates, so an exact numeric is the sensible default; +// dimensions, in schemaAspectData, default to STRING) rather than emit an +// invalid aspect. function metricAspectData( - metric: Metric, warnings: string[], - emitExpressions: boolean): Record { + metric: Metric, warnings: string[]): Record { let dataType = metric.type ? columnDataType(metric.type) : undefined; if (!dataType) { warnings.push( @@ -328,9 +372,6 @@ function metricAspectData( return compact({ entity: metric.entity, dataType, - // `expression` is not in the published semantic-metric aspect template yet; - // gated off by default (see KcGenerateOptions.emitExpressions). - expression: emitExpressions ? metric.expression : undefined, }); } diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index a1ebae76..13cff782 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -152,8 +152,11 @@ export async function pullKnowledgeCatalog( // The aspect type resource names to hydrate for a semantic entry, derived from // its entryType (the aspect types are the parallel resources in the same // project/location). An entity carries two aspects: its `semantic-entity` -// aspect and the built-in `schema` aspect that holds its fields. Returns -// undefined for entries that are not part of a semantic model. +// aspect and the built-in `schema` aspect that holds its fields. Entities and +// metrics also carry the optional `sql-expressions` companion aspect (the +// field/metric expressions); it is absent on a default push, so requesting it +// is harmless when it does not exist. Returns undefined for entries that are +// not part of a semantic model. function semanticAspectTypes(entryType: string): string[]|undefined { const marker = '/entryTypes/'; const idx = entryType?.indexOf(marker) ?? -1; @@ -165,9 +168,12 @@ function semanticAspectTypes(entryType: string): string[]|undefined { case 'semantic-model': return [aspectType('semantic-model')]; case 'semantic-entity': - return [aspectType('semantic-entity'), aspectType('schema')]; + return [ + aspectType('semantic-entity'), aspectType('schema'), + aspectType('sql-expressions') + ]; case 'semantic-metric': - return [aspectType('semantic-metric')]; + return [aspectType('semantic-metric'), aspectType('sql-expressions')]; default: return undefined; } diff --git a/toolbox/mdcode/src/libts/semantic/sql_expressions.ts b/toolbox/mdcode/src/libts/semantic/sql_expressions.ts new file mode 100644 index 00000000..d8a6acf1 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/sql_expressions.ts @@ -0,0 +1,103 @@ +// The `sql-expressions` companion aspect: the shared codec both KC push and +// pull use to move SQL out of the metadata-only system aspects. +// +// Per the V2 "SQL Expression Storage in USL Semantic Models" proposal +// (D. Lychagin, 2026-08-13), executable SQL is NOT stored on the metadata-only +// `schema` / `semantic-metric` aspects. It lives in a single, general-purpose +// `sql-expressions` aspect (data_classification: METADATA_AND_DATA) attached to +// the same entry, so catalog metadata discovery stays open while the SQL is +// governed behind data-level access. This module is the one place that fixes +// the aspect's shape and its qualifier convention; the emitter +// (`knowledge_catalog.ts`) and the reader (`kc_converter.ts`) are exact +// inverses over it, which is what keeps a push -> pull round trip lossless. +// +// Shape (mirroring `descriptions.textproto`'s dual-level pattern): +// { +// expressions?: [{ qualifier?, sql }, ...] // entry-level (metrics) +// fields?: [{ name, expressions: [{ qualifier?, sql }, ...] }, ...] // per +// field +// } +// +// Qualifier convention (open taxonomy server-side; the subset we author): +// * omitted -> the primary / canonical GoogleSQL expression (IR +// `expression`) +// * "imported" -> the source-dialect expression, verbatim (IR +// `importedExpression`) +// The specific source dialect is intentionally NOT stored here (the proposal +// infers it from the entity's `importedSystem`); an imported record therefore +// reads back as `importedExpression` with the dialect left for a later pass. + +// Bare aspect-type id, matched by suffix on read so the system-type +// project/location need not be known (see kc_converter.aspectDataOf). +export const SQL_EXPRESSIONS_ASPECT = 'sql-expressions'; + +// The one qualifier value we author beyond the (omitted) primary expression. +export const IMPORTED_QUALIFIER = 'imported'; + +// One SQL expression record: the raw SQL plus an optional role/dialect +// qualifier. `qualifier` is omitted for the primary GoogleSQL expression. +export interface SqlExpressionRecord { + qualifier?: string; + sql: string; +} + +// One field's expression records, bound to the schema field of the same name. +export interface SqlFieldExpressions { + name: string; + expressions: SqlExpressionRecord[]; +} + +// The `sql-expressions` aspect data payload. Both arrays are optional; the +// aspect is only attached when at least one is non-empty. +export interface SqlExpressionsData { + expressions?: SqlExpressionRecord[]; + fields?: SqlFieldExpressions[]; +} + +/** + * The expression records for one IR element (field or metric): the primary + * GoogleSQL form (unqualified) and/or the imported vendor form (qualifier + * `imported`), in that order. Returns [] when neither form is set, so a caller + * can decide whether to attach the aspect at all. + */ +export function expressionRecords( + expression: string|undefined, + importedExpression: string|undefined): SqlExpressionRecord[] { + const records: SqlExpressionRecord[] = []; + if (expression !== undefined) records.push({sql: expression}); + if (importedExpression !== undefined) { + records.push({qualifier: IMPORTED_QUALIFIER, sql: importedExpression}); + } + return records; +} + +// The IR expression forms recovered from a records array. +export interface RecoveredExpressions { + expression?: string; + importedExpression?: string; +} + +/** + * The inverse of `expressionRecords`: recovers the primary and imported forms + * from an expression-records array. A record whose `qualifier` is omitted (or + * empty) is the primary GoogleSQL expression; a record qualified `imported` is + * the vendor form. The source dialect is not stored, so only `importedExpression` + * (not its dialect) comes back. The first record of each kind wins; malformed + * records (no `sql`) and records with any other qualifier are ignored. + */ +export function recoverExpressions(records: SqlExpressionRecord[]|undefined): + RecoveredExpressions { + const out: RecoveredExpressions = {}; + for (const rec of records ?? []) { + if (!rec || typeof rec.sql !== 'string') continue; + if (rec.qualifier === IMPORTED_QUALIFIER) { + if (out.importedExpression === undefined) + out.importedExpression = rec.sql; + } else if (rec.qualifier === undefined || rec.qualifier === '') { + if (out.expression === undefined) out.expression = rec.sql; + } + // Any other qualifier (future filter/sql_on/... roles) is not part of the + // field/metric expression model yet and is skipped rather than guessed. + } + return out; +} diff --git a/toolbox/mdcode/src/tool/commands.ts b/toolbox/mdcode/src/tool/commands.ts index 1c67d76f..fe7d1566 100644 --- a/toolbox/mdcode/src/tool/commands.ts +++ b/toolbox/mdcode/src/tool/commands.ts @@ -52,10 +52,11 @@ export interface PushOptions { // plan), each block labeled by destination. Scope which destinations run with // --target. Works with or without --validate-only. Semantic-model push only. print?: boolean; - // Emit the SQL-expression fields not yet supported by the published Knowledge - // Catalog system-type templates (per-field schema semantics and the metric - // expression). Off by default so a push matches the live types; enable once - // the templates gain these fields. Semantic-model KC push only. + // Emit the sql-expressions companion aspect (field/metric expressions, primary + // + imported) and the schema DIMENSION marker, not yet supported by the + // published Knowledge Catalog system-type templates. Off by default so a push + // matches the live types; enable once the sql-expressions aspect type is + // provisioned. Semantic-model KC push only. emitExpressions?: boolean; } diff --git a/toolbox/mdcode/src/tool/main.ts b/toolbox/mdcode/src/tool/main.ts index 84cceff9..0ab29f1d 100644 --- a/toolbox/mdcode/src/tool/main.ts +++ b/toolbox/mdcode/src/tool/main.ts @@ -43,7 +43,7 @@ cli.command('pull', 'Pull catalog entries') cli.command('push', 'Push catalog entries') .option('--force', 'Force push changes') .option('--force-remove', 'Delete Knowledge Catalog models in the entry group that this push does not include (removed/renamed models); semantic-model push only') - .option('--emit-expressions', 'Emit SQL-expression fields not yet in the published Knowledge Catalog system-type templates (per-field schema semantics, metric expression); off by default, enable once the templates support them; semantic-model push only') + .option('--emit-expressions', 'Emit the sql-expressions companion aspect (field/metric expressions, primary + imported) and the schema DIMENSION marker, not yet in the published Knowledge Catalog system-type templates; off by default, enable once the sql-expressions aspect type is provisioned; semantic-model push only') .option('--validate-only', 'Only validate changes without applying') .option('--target ', 'Semantic-model push destination(s): bq, kc, all (default), or a comma-separated list (e.g. bq,kc)') .option('--print', 'Print each pushed destination\'s generated artifact in its native format (BigQuery Graph SQL DDL, Knowledge Catalog entry plan); scope with --target (semantic-model push only)') diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 4ccac618..1c1d39e3 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -5,18 +5,21 @@ // central guarantee is an emitter -> reader round trip: emit a model's entries // AND entry links, read them back, and get an IR equal to the source WHERE the // emitter is lossless. The write drops content by design (entity keys, -// ai_context, field labels, importedDialect, and many-to-many relationships -- -// see the emitter header), so the expected read-back is the source model with -// exactly those fields cleared. It ALSO drops the per-field `semantics` block -// (field/metric expressions and the DIMENSION role) unless the push enabled -// `emitExpressions`, so a DEFAULT round trip (roundTrip) loses those too, while -// a full round trip (roundTripFull, emitExpressions: true) keeps them. 1:1 / -// 1:N relationships and deployment targets DO round-trip either way (via -// schema-join links and the semantic-model aspect), except that relationship -// names come back normalized (lowercased/hyphenated). Targeted tests pin the -// mapping details a round trip cannot isolate (the dataType inverse, the -// DIMENSION role, resource-URI parsing, metric attach re-derivation, -// relationship endpoint/direction recovery, and parent/anchor grouping). +// ai_context, field labels, the imported dialect label, and many-to-many +// relationships -- see the emitter header), so the expected read-back is the +// source model with exactly those fields cleared. It ALSO omits the +// sql-expressions companion aspect (field/metric expressions -- primary AND the +// imported vendor form) and the schema field's DIMENSION `semantic` marker +// unless the push enabled `emitExpressions`, so a DEFAULT round trip +// (roundTrip) loses those too, while a full round trip (roundTripFull, +// emitExpressions: true) keeps them (importedExpression included; only its +// dialect label is lost). 1:1 / 1:N relationships and deployment targets DO +// round-trip either way (via schema-join links and the semantic-model aspect), +// except that relationship names come back normalized (lowercased/hyphenated). +// Targeted tests pin the mapping details a round trip cannot isolate (the +// dataType inverse, the DIMENSION role, resource-URI parsing, metric attach +// re-derivation, relationship endpoint/direction recovery, and parent/anchor +// grouping). import {describe, expect, test} from 'bun:test'; import * as fs from 'node:fs'; @@ -37,8 +40,9 @@ const OPTS = { }; // Emits a model to entries + entry links and reads it straight back. The -// default push omits the per-field `semantics` block (expressions + role), so a -// default round trip drops those; use roundTripFull to exercise their recovery. +// default push omits the sql-expressions companion aspect (field/metric +// expressions) and the schema DIMENSION marker, so a default round trip drops +// those; use roundTripFull to exercise their recovery. function roundTrip(model: SemanticModel): {models: SemanticModel[]; warnings: string[]} { const {entries, entryLinks} = generateCatalogResources(model, OPTS); @@ -46,8 +50,9 @@ function roundTrip(model: SemanticModel): } // A round trip through a `--emit-expressions` push: the catalog then holds the -// per-field `semantics` block and the metric expression, so field/metric -// expressions and DIMENSION roles round-trip too. +// sql-expressions companion aspect and the schema DIMENSION marker, so +// field/metric expressions (primary + imported) and DIMENSION roles round-trip +// too. function roundTripFull(model: SemanticModel): {models: SemanticModel[]; warnings: string[]} { const {entries, entryLinks} = @@ -61,7 +66,7 @@ describe('emitter -> reader round trip (lossless slice)', () => { // relationships (all dropped by the write), and datatypes that invert // cleanly. Its fields and metric carry expressions and a DIMENSION role, so // it round-trips losslessly only through a `--emit-expressions` push - // (roundTripFull); a default push omits the per-field `semantics` block. + // (roundTripFull); a default push omits the sql-expressions companion aspect. const source: SemanticModel = { name: 'sales', description: 'the sales model', @@ -385,14 +390,14 @@ describe('field mapping details', () => { }; return rt(model).models[0].entities[0].fields[0]; } - // Default push (no per-field `semantics`) vs a `--emit-expressions` push. + // Default push (no sql-expressions aspect) vs a `--emit-expressions` push. const readField = (f: Entity['fields'][number]) => readWith(roundTrip, f); const readFieldFull = (f: Entity['fields'][number]) => readWith(roundTripFull, f); test('a DIMENSION role reads back as a dimension marker', () => { - // The role lives in the gated `semantics` block, so it survives only a - // `--emit-expressions` push. + // The role lives in the gated schema `semantic` marker, so it survives only + // a `--emit-expressions` push. const back = readFieldFull({name: 'd', expression: 'e.d', dimension: {}}); expect(back.dimension).toEqual({}); }); @@ -403,7 +408,7 @@ describe('field mapping details', () => { }); test( - 'a default push drops the DIMENSION role with the semantics block', + 'a default push drops the expression + DIMENSION role (no companion aspect)', () => { const back = readField({name: 'd', expression: 'e.d', dimension: {}}); expect(back.dimension).toBeUndefined(); @@ -411,7 +416,7 @@ describe('field mapping details', () => { }); test( - 'an imported expression is not persisted to or recovered from the catalog', + 'an imported expression round-trips via the sql-expressions aspect', () => { const back = readFieldFull({ name: 'amt', @@ -420,9 +425,10 @@ describe('field mapping details', () => { importedDialect: 'SNOWFLAKE', }); expect(back.expression).toBe('e.amt'); - // The emitter no longer writes importedExpression/importedDialect, so - // neither survives the round trip. - expect(back.importedExpression).toBeUndefined(); + // The companion aspect stores the imported form under an `imported` + // qualifier, so importedExpression survives; its source dialect is not + // stored, so importedDialect does not. + expect(back.importedExpression).toBe('e.amt::NUMBER'); expect(back.importedDialect).toBeUndefined(); }); @@ -722,12 +728,14 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { for (const f of e.fields) { delete f.label; // display label is not persisted delete f.aiContext; - delete f.importedExpression; // vendor SQL is not persisted + delete f.importedExpression; // dropped by a default push (no companion + // aspect) delete f.importedDialect; delete f.customExtensions; - // A default push omits the per-field `semantics` block, so the field - // expression and the DIMENSION role are not persisted (they ride back - // only through a `--emit-expressions` push). + // A default push omits the sql-expressions companion aspect and the + // schema DIMENSION marker, so the field expression and the DIMENSION role + // are not persisted (they ride back only through a `--emit-expressions` + // push). delete f.expression; delete f.dimension; // String is indistinguishable from an un-typed field on read. diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts index 621f402a..df871c5c 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -24,9 +24,13 @@ const OPTS = { location: 'us', entryGroup: 'eg' }; -// The expression fields (per-field schema semantics, metric expression) are -// gated off by default; this turns them on to assert their content. -const OPTS_EXPR = {...OPTS, emitExpressions: true}; +// SQL expressions (the sql-expressions companion aspect + the schema field's +// DIMENSION `semantic` marker) are gated off by default; this turns them on to +// assert their content. +const OPTS_EXPR = { + ...OPTS, + emitExpressions: true +}; // A one-entity model whose single field carries the given IR type + dimension, // so a test can read back the emitted schema aspect for that field. @@ -46,11 +50,34 @@ function modelWithField( return {name: 'm', entities: [entity], relationships: [], metrics: []}; } -// The schema-aspect field record for the sole field of modelWithField. -function schemaField( +// The aspect-map keys on the sole entity entry (to assert presence/absence of +// the sql-expressions companion aspect). +function entityAspectKeys(model: SemanticModel, opts = OPTS): string[] { + const {entries} = generateCatalogResources(model, opts); + const entity = entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + return Object.keys(entity.aspects ?? {}); +} + +// The sql-expressions companion aspect data on the sole entity entry. +function entitySqlExpressions( model: SemanticModel, opts = OPTS): Record { const {entries} = generateCatalogResources(model, opts); const entity = entries.find(e => e.entryType.endsWith('/semantic-entity'))!; + return entity.aspects!['dataplex-types.global.sql-expressions'].data!; +} + +// The sql-expressions companion aspect data on the sole metric entry. +function metricSqlExpressions( + model: SemanticModel, opts = OPTS): Record { + const {entries} = generateCatalogResources(model, opts); + const metric = entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + return metric.aspects!['dataplex-types.global.sql-expressions'].data!; +} + +// The schema-aspect field record for the sole field of modelWithField. +function schemaField(model: SemanticModel, opts = OPTS): Record { + const {entries} = generateCatalogResources(model, opts); + const entity = entries.find(e => e.entryType.endsWith('/semantic-entity'))!; const schema = entity.aspects!['dataplex-types.global.schema'].data!; return schema.fields[0]; } @@ -88,56 +115,100 @@ describe( describe( - 'a field with dimension metadata gets role DIMENSION, else DEFAULT', () => { + 'a dimension field gets schema.semantic DIMENSION, else it is unset', + () => { test('dimension -> DIMENSION', () => { - expect(schemaField(modelWithField('String', true), OPTS_EXPR).semantics.role) + expect(schemaField(modelWithField('String', true), OPTS_EXPR).semantic) .toBe('DIMENSION'); }); - test('no dimension -> DEFAULT', () => { - expect(schemaField(modelWithField('String', false), OPTS_EXPR).semantics.role) - .toBe('DEFAULT'); + test('no dimension -> semantic unset', () => { + expect(schemaField(modelWithField('String', false), OPTS_EXPR).semantic) + .toBeUndefined(); }); }); -describe('the schema aspect carries the target expression in semantics', () => { - test('the target expression is kept; imported vendor SQL is not emitted', () => { - const model: SemanticModel = { - name: 'm', - relationships: [], - metrics: [], - entities: [{ - name: 'e', - dataSource: 'p.d.t', - keys: ['k'], - fields: [{ - name: 'f', - expression: 'CAST(e.f AS INT64)', - importedExpression: 'e.f::int', - importedDialect: 'SNOWFLAKE', - }], - }], - }; - const f = schemaField(model, OPTS_EXPR); - expect(f.semantics.expression).toBe('CAST(e.f AS INT64)'); - // importedExpression is the vendor/MAQL form; KC has no consumer for it. - expect(f.semantics.importedExpression).toBeUndefined(); - }); -}); +describe( + 'field expressions live in the sql-expressions companion aspect', () => { + test( + 'both the primary GoogleSQL and the imported vendor form are emitted', + () => { + const model: SemanticModel = { + name: 'm', + relationships: [], + metrics: [], + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{ + name: 'f', + expression: 'CAST(e.f AS INT64)', + importedExpression: 'e.f::int', + importedDialect: 'SNOWFLAKE', + }], + }], + }; + // The schema aspect stays metadata-only: no expression on the + // field. + const sf = schemaField(model, OPTS_EXPR); + expect(sf.semantics).toBeUndefined(); + expect(sf.expression).toBeUndefined(); + // The expressions ride in the companion aspect: the primary + // (unqualified) GoogleSQL form + the imported vendor form under the + // `imported` qualifier. + expect(entitySqlExpressions(model, OPTS_EXPR).fields).toEqual([{ + name: 'f', + expressions: [ + {sql: 'CAST(e.f AS INT64)'}, + {qualifier: 'imported', sql: 'e.f::int'}, + ], + }]); + }); + }); -describe('the schema semantics block is gated behind emitExpressions', () => { - test('omitted by default; the non-gated columns are still emitted', () => { - const f = schemaField(modelWithField('String', true)); - expect(f.semantics).toBeUndefined(); - expect(f.dataType).toBe('STRING'); - expect(f.metadataType).toBe('STRING'); - }); - test('emitExpressions re-adds the semantics block (expression + role)', () => { - const f = schemaField(modelWithField('String', true), OPTS_EXPR); - expect(f.semantics).toEqual({expression: 'e.f', role: 'DIMENSION'}); - }); -}); +describe( + 'the companion aspect + schema.semantic are gated behind emitExpressions', + () => { + test( + 'omitted by default; the non-gated columns are still emitted', () => { + const model = modelWithField('String', true); + const f = schemaField(model); + expect(f.semantic).toBeUndefined(); + expect(f.dataType).toBe('STRING'); + expect(f.metadataType).toBe('STRING'); + // No sql-expressions aspect on a default push. + expect(entityAspectKeys(model).some( + k => k.endsWith('.sql-expressions'))) + .toBe(false); + }); + test( + 'emitExpressions adds the companion aspect + the DIMENSION marker', + () => { + const model = modelWithField('String', true); + expect(schemaField(model, OPTS_EXPR).semantic).toBe('DIMENSION'); + expect(entitySqlExpressions(model, OPTS_EXPR).fields).toEqual([ + {name: 'f', expressions: [{sql: 'e.f'}]}, + ]); + }); + test('no companion aspect when no field has an expression', () => { + const model: SemanticModel = { + name: 'm', + relationships: [], + metrics: [], + entities: [{ + name: 'e', + dataSource: 'p.d.t', + keys: ['k'], + fields: [{name: 'f'}], // no expression + }], + }; + expect(entityAspectKeys(model, OPTS_EXPR) + .some(k => k.endsWith('.sql-expressions'))) + .toBe(false); + }); + }); describe('entity dataSource maps to a resource path', () => { @@ -192,8 +263,8 @@ describe('semantic-metric aspect', () => { ], }; const {data, warnings} = metricData(model, OPTS_EXPR); - expect(data).toEqual( - {entity: 'o', dataType: 'NUMERIC', expression: 'SUM(o.p)'}); + // The metric aspect stays metadata-only: no expression here. + expect(data).toEqual({entity: 'o', dataType: 'NUMERIC'}); expect(warnings.some(w => w.includes('dataType'))).toBe(false); }); @@ -212,19 +283,35 @@ describe('semantic-metric aspect', () => { .toBe(true); }); - test('the expression is gated: omitted by default, kept with emitExpressions', - () => { - const model: SemanticModel = { - name: 'm', - entities: [], - relationships: [], - metrics: [{ - name: 'rev', expression: 'SUM(o.p)', entity: 'o', type: 'Decimal' - }], - }; - expect(metricData(model).data).toEqual({entity: 'o', dataType: 'NUMERIC'}); - expect(metricData(model, OPTS_EXPR).data.expression).toBe('SUM(o.p)'); - }); + test( + 'the aggregate formula rides in the sql-expressions companion aspect, gated', + () => { + const model: SemanticModel = { + name: 'm', + entities: [], + relationships: [], + metrics: [{ + name: 'rev', + expression: 'SUM(o.p)', + importedExpression: 'SUM(o.p)::NUMBER', + importedDialect: 'SNOWFLAKE', + entity: 'o', + type: 'Decimal' + }], + }; + // Default push: no companion aspect at all. + const def = generateCatalogResources(model, OPTS); + const defMetric = + def.entries.find(e => e.entryType.endsWith('/semantic-metric'))!; + expect(Object.keys(defMetric.aspects ?? {}) + .some(k => k.endsWith('.sql-expressions'))) + .toBe(false); + // --emit-expressions push: primary GoogleSQL + imported vendor form. + expect(metricSqlExpressions(model, OPTS_EXPR).expressions).toEqual([ + {sql: 'SUM(o.p)'}, + {qualifier: 'imported', sql: 'SUM(o.p)::NUMBER'}, + ]); + }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts index 83f5aee0..102eb2ef 100644 --- a/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/pull_kc.test.ts @@ -116,7 +116,8 @@ describe('pullKnowledgeCatalog: happy path', () => { }); test( - 'an entity is hydrated with BOTH its semantic-entity and schema aspects', + 'an entity is hydrated with its semantic-entity, schema, and ' + + 'sql-expressions aspects', async () => { const entries = entriesFor(SALES); const {lookup} = stubClient(entries, entries); @@ -136,6 +137,25 @@ describe('pullKnowledgeCatalog: happy path', () => { .toBe(true); expect(aspectTypes.some(t => t.endsWith('/aspectTypes/schema'))) .toBe(true); + // The companion sql-expressions aspect is hydrated too, so field + // expressions come back. + expect( + aspectTypes.some(t => t.endsWith('/aspectTypes/sql-expressions'))) + .toBe(true); + }); + + test( + 'a --emit-expressions push round-trips field + metric expressions', + async () => { + const entries = entriesFor(SALES); + stubClient(entries, entries); + + const cat = new CatalogClient({} as any); + const {models} = await pullKnowledgeCatalog(cat, OPTS); + const model = models[0]; + expect(model.entities[0].fields[0].expression) + .toBe('orders.o_totalprice'); + expect(model.metrics[0].expression).toBe('SUM(orders.o_totalprice)'); }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/sql_expressions.test.ts b/toolbox/mdcode/tests/libts/semantic/sql_expressions.test.ts new file mode 100644 index 00000000..01555217 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/sql_expressions.test.ts @@ -0,0 +1,79 @@ +// Behavior specification for the sql-expressions aspect codec +// (src/libts/semantic/sql_expressions.ts). expressionRecords (emit) and +// recoverExpressions (read) are exact inverses over the qualifier convention: +// omitted -> primary GoogleSQL, `imported` -> the vendor form. + +import {describe, expect, test} from 'bun:test'; + +import {expressionRecords, recoverExpressions} from '../../../src/libts/semantic/sql_expressions'; + +describe('expressionRecords (emit)', () => { + test('primary only -> a single unqualified record', () => { + expect(expressionRecords('SUM(o.p)', undefined)).toEqual([ + {sql: 'SUM(o.p)'} + ]); + }); + + test('imported only -> a single `imported`-qualified record', () => { + expect(expressionRecords(undefined, 'o.p::NUMBER')).toEqual([ + {qualifier: 'imported', sql: 'o.p::NUMBER'}, + ]); + }); + + test('both -> primary first, then imported', () => { + expect(expressionRecords('CAST(o.p AS INT64)', 'o.p::int')).toEqual([ + {sql: 'CAST(o.p AS INT64)'}, + {qualifier: 'imported', sql: 'o.p::int'}, + ]); + }); + + test('neither -> empty (caller omits the aspect)', () => { + expect(expressionRecords(undefined, undefined)).toEqual([]); + }); + + test('an empty-string expression is a value, not absence', () => { + expect(expressionRecords('', undefined)).toEqual([{sql: ''}]); + }); +}); + + +describe('recoverExpressions (read)', () => { + test('inverts expressionRecords for the both-forms case', () => { + const records = expressionRecords('CAST(o.p AS INT64)', 'o.p::int'); + expect(recoverExpressions(records)).toEqual({ + expression: 'CAST(o.p AS INT64)', + importedExpression: 'o.p::int', + }); + }); + + test('an unqualified record is the primary expression', () => { + expect(recoverExpressions([{sql: 'o.p'}])).toEqual({expression: 'o.p'}); + }); + + test('an `imported` record is the imported expression only', () => { + expect(recoverExpressions([{qualifier: 'imported', sql: 'o.p::int'}])) + .toEqual({importedExpression: 'o.p::int'}); + }); + + test('missing / empty records recover nothing', () => { + expect(recoverExpressions(undefined)).toEqual({}); + expect(recoverExpressions([])).toEqual({}); + }); + + test('the first record of each kind wins', () => { + expect(recoverExpressions([ + {sql: 'first'}, + {sql: 'second'}, + {qualifier: 'imported', sql: 'imp1'}, + {qualifier: 'imported', sql: 'imp2'}, + ])).toEqual({expression: 'first', importedExpression: 'imp1'}); + }); + + test('malformed records (no sql) and unknown qualifiers are skipped', () => { + expect(recoverExpressions([ + {qualifier: 'sql_always_where', sql: 'x = 1'}, // future role, not ours + {sql: undefined as any}, // malformed + {sql: 'o.p'}, + ])).toEqual({expression: 'o.p'}); + }); +}); From b8759988776f74c63c8ba7ce2d5eb39aa4edf7f7 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Fri, 14 Aug 2026 06:12:17 +0000 Subject: [PATCH 14/14] mdcode: green the OSI guardrail and show expressions in pull goldens Code-review follow-ups on the sql-expressions aspect change. - Generate each corpus pull golden from a --emit-expressions round trip instead of the default-push emitter JSON, so the reviewable .pull.golden.yaml artifacts exhibit the recovered field/metric expressions and the corpus golden-pull test exercises that path. - Exclude the lossy KC pull reconstructions from the OSI schema guardrail: a KC round trip does not persist an imported expression's source dialect, so a recovered vendor form serializes under the `IMPORTED` placeholder, which is outside the OSI dialect enum. The pull goldens' exact content and documented losses stay pinned by kc_converter.test.ts; the guardrail keeps validating the authored fixtures and the full-fidelity OSI-converter goldens, the artifacts that must be valid OSI. - Fix a dead assertion: sf.semantics -> sf.semantic (the schema field was renamed from the plural `semantics` block to a `semantic` marker). --- .../sales_bq_graph_target.pull.golden.yaml | 12 ++ .../star_orders_customer.pull.golden.yaml | 33 ++++++ .../fixtures/tpcds_date_edge.pull.golden.yaml | 110 ++++++++++++++++++ .../tests/libts/semantic/kc_converter.test.ts | 39 ++++--- .../libts/semantic/knowledge_catalog.test.ts | 7 +- .../tests/libts/semantic/osi_schema.test.ts | 12 +- 6 files changed, 194 insertions(+), 19 deletions(-) diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml index f966f619..06dd8bdc 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/sales_bq_graph_target.pull.golden.yaml @@ -10,7 +10,19 @@ semantic_model: source: demo.sales.orders fields: - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice metrics: - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) datatype: Decimal diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml index ceb6bed1..d7a9cbe1 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -12,15 +12,40 @@ semantic_model: description: One row per order fields: - name: o_orderkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderkey description: Order identifier - name: o_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: o_custkey - name: o_orderdate + expression: + dialects: + - dialect: BIGQUERY + expression: o_orderdate + dimension: {} - name: o_totalprice + expression: + dialects: + - dialect: BIGQUERY + expression: o_totalprice - name: customer source: samples.tpch.customer fields: - name: c_custkey + expression: + dialects: + - dialect: BIGQUERY + expression: c_custkey - name: c_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_name description: Customer name relationships: - name: orders-to-customer @@ -32,8 +57,16 @@ semantic_model: - c_custkey metrics: - name: total_revenue + expression: + dialects: + - dialect: BIGQUERY + expression: SUM(orders.o_totalprice) datatype: Decimal description: Total order revenue - name: order_count + expression: + dialects: + - dialect: BIGQUERY + expression: COUNT(orders.o_orderkey) datatype: Decimal description: Number of orders diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml index ab45bace..f4e5dc41 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -9,44 +9,154 @@ semantic_model: description: Fact table containing all store sales transactions fields: - name: ss_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_item_sk + - dialect: IMPORTED + expression: "{label/store_sales.attr.store_sales.ss_item_sk}" + dimension: {} - name: ss_ticket_number + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ticket_number + - dialect: IMPORTED + expression: "{label/store_sales.attr.store_sales.ss_ticket_number}" + dimension: {} - name: ss_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_customer_sk + - dialect: IMPORTED + expression: "{label/store_sales.attr.store_sales.ss_customer_sk}" + dimension: {} - name: ss_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: ss_store_sk + - dialect: IMPORTED + expression: "{label/store_sales.attr.store_sales.ss_store_sk}" + dimension: {} - name: ss_quantity + expression: + dialects: + - dialect: BIGQUERY + expression: ss_quantity + - dialect: IMPORTED + expression: "{fact/store_sales.fact.store_sales.ss_quantity}" description: Quantity of items sold - name: ss_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_sales_price + - dialect: IMPORTED + expression: "{fact/store_sales.fact.store_sales.ss_sales_price}" description: Sales price per unit - name: ss_ext_sales_price + expression: + dialects: + - dialect: BIGQUERY + expression: ss_ext_sales_price + - dialect: IMPORTED + expression: "{fact/store_sales.fact.store_sales.ss_ext_sales_price}" description: Extended sales price (quantity * price) - name: ss_net_profit + expression: + dialects: + - dialect: BIGQUERY + expression: ss_net_profit + - dialect: IMPORTED + expression: "{fact/store_sales.fact.store_sales.ss_net_profit}" description: Net profit from the sale - name: customer source: tpcds.public.customer description: Customer dimension with demographic information fields: - name: c_customer_sk + expression: + dialects: + - dialect: BIGQUERY + expression: c_customer_sk + dimension: {} - name: c_first_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_first_name + dimension: {} description: Customer first name - name: c_last_name + expression: + dialects: + - dialect: BIGQUERY + expression: c_last_name + dimension: {} description: Customer last name - name: item source: tpcds.public.item description: Item/Product dimension fields: - name: i_item_sk + expression: + dialects: + - dialect: BIGQUERY + expression: i_item_sk + dimension: {} - name: i_brand + expression: + dialects: + - dialect: BIGQUERY + expression: i_brand + dimension: {} - name: i_category + expression: + dialects: + - dialect: BIGQUERY + expression: i_category + dimension: {} - name: i_current_price + expression: + dialects: + - dialect: BIGQUERY + expression: i_current_price description: Current price of the item - name: store source: tpcds.public.store description: Store dimension with location attributes fields: - name: s_store_sk + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_sk + dimension: {} - name: s_store_name + expression: + dialects: + - dialect: BIGQUERY + expression: s_store_name + dimension: {} - name: s_city + expression: + dialects: + - dialect: BIGQUERY + expression: s_city + dimension: {} - name: s_state + expression: + dialects: + - dialect: BIGQUERY + expression: s_state + dimension: {} - name: s_number_employees + expression: + dialects: + - dialect: BIGQUERY + expression: s_number_employees description: Number of employees at the store - name: date_dim source: sqlgen-testing.demo.date_dim diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 1c1d39e3..4848ac5a 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -610,38 +610,47 @@ describe('metric expression referencing no known entity', () => { }); -// -- Golden pull: the whole KC entries -> IR -> OSI YAML output. -- +// -- Golden pull: the whole model -> KC entries -> IR -> OSI YAML output. -- // // The round trip above proves the reader inverts the emitter in memory; this -// pins the reviewable artifact. For each corpus fixture it reads the committed -// emitter golden (`.knowledge_catalog.golden.json` -- the exact -// entries and entry links a push produced) back through the reader and -// serializes the reconstructed IR to `.pull.golden.yaml`. Open that -// next to the fixture's `.osi.golden.yaml` to see, as whole files, what a -// Knowledge Catalog round trip preserves (including 1:1 / 1:N relationships and -// deployment targets) and what it drops (keys, ai_context, labels, vendor SQL, -// M:N relationships). +// pins the reviewable artifact. For each corpus fixture it emits the model +// through a `--emit-expressions` push (so the catalog carries the +// sql-expressions companion aspect) and reads the entries and entry links back +// through the reader, serializing the reconstructed IR to +// `.pull.golden.yaml`. Open that next to the fixture's +// `.osi.golden.yaml` to see, as whole files, what a Knowledge Catalog round +// trip preserves (field/metric expressions, 1:1 / 1:N relationships, deployment +// targets) and what it drops (keys, ai_context, labels, the imported dialect +// label, M:N relationships). The primary GoogleSQL and the imported vendor form +// both come back; the imported form's dialect is not persisted, so it serializes +// under the `IMPORTED` placeholder -- which is why osi_schema.test.ts excludes +// these lossy reconstructions from its OSI dialect-enum check. // // Regenerate after an intentional reader/serializer change: // UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts describe( - 'golden pull: each corpus KC golden reconstructs to its exact YAML', () => { + 'golden pull: each corpus model reconstructs to its exact YAML', () => { const CORPUS = [ 'sales_bq_graph_target.yaml', 'star_orders_customer.yaml', 'tpcds_date_edge.yaml', ]; - const kcGoldenPath = (fixture: string) => path.join( - FIXTURES, - fixture.replace(/\.yaml$/, '.knowledge_catalog.golden.json')); + // Same load defaults as the symmetry / OSI goldens, so the pulled model + // lines up with the fixture's other goldens. + const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; const pullGoldenPath = (fixture: string) => path.join(FIXTURES, fixture.replace(/\.yaml$/, '.pull.golden.yaml')); for (const fixture of CORPUS) { test(fixture, () => { - const kc = JSON.parse(fs.readFileSync(kcGoldenPath(fixture), 'utf8')); + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + const [model] = loadModels(text, LOAD).models; + // Emit through a --emit-expressions push so the round trip carries + // the sql-expressions companion aspect (see roundTripFull). + const {entries, entryLinks} = + generateCatalogResources(model, {...OPTS, emitExpressions: true}); const {models, warnings} = - modelsFromCatalogResources(kc.entries, kc.entryLinks ?? []); + modelsFromCatalogResources(entries, entryLinks); // Reader warnings ride along as YAML comments so the golden shows // the full outcome, not just the recovered document. const header = warnings.length ? diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts index df871c5c..4c6c4b0d 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -149,10 +149,11 @@ describe( }], }], }; - // The schema aspect stays metadata-only: no expression on the - // field. + // The schema aspect stays metadata-only: no inline expression on + // the field (the field is not a dimension, so `semantic` is unset + // too). const sf = schemaField(model, OPTS_EXPR); - expect(sf.semantics).toBeUndefined(); + expect(sf.semantic).toBeUndefined(); expect(sf.expression).toBeUndefined(); // The expressions ride in the companion aspect: the primary // (unqualified) GoogleSQL form + the imported vendor form under the diff --git a/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts index 2249ef0f..f4777a53 100644 --- a/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts @@ -35,7 +35,17 @@ function yamlFixtures(dir: string): string[] { // against it, not the schema's own meta-style. const ajv = new Ajv2020({ allErrors: true, strict: false }); const validate = ajv.compile(schema); -const fixtures = yamlFixtures(fixturesDir); + +// The `.pull.golden.yaml` artifacts are lossy Knowledge Catalog reconstructions, +// not authored OSI documents: a KC round trip does not persist an imported +// expression's source dialect (see sql_expressions.ts -- the dialect is inferred +// from the entity, not stored), so a pulled model that carried a vendor form +// serializes it under the `IMPORTED` placeholder dialect, which is outside the +// OSI dialect enum. Their exact content and documented losses are pinned by +// kc_converter.test.ts; this guardrail validates the authored input fixtures and +// the full-fidelity OSI-converter goldens, the artifacts that must be valid OSI. +const fixtures = + yamlFixtures(fixturesDir).filter(p => !p.endsWith('.pull.golden.yaml')); describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => { test('at least one fixture is discovered', () => {