From 96d20388da7ac948733bb5519743db5a49d2500f Mon Sep 17 00:00:00 2001 From: Bei Li Date: Tue, 8 Sep 2026 16:04:37 +0000 Subject: [PATCH 1/3] mdcode: make many-to-many relationships authorable A many-to-many edge is backed by a junction table rather than by a foreign key on either endpoint. The IR has modelled that (`Association`) since the BigQuery graph generator gained it, and the Spanner generator renders it too, but there was no way to write one down: the format's relationship schema knew only the direct foreign key, so the only models that reached those generators were hand-built IR in tests. Adds an `association` block on a relationship, a native key of the extended profile ('0.2.0.dev0/google'). It names the junction table, the edge's own key, the junction columns that reference each endpoint's key, and the properties of the pairing itself. The relationship's own from_columns/to_columns must be absent when it is present -- the two are alternative bindings, and neither endpoint holds a foreign key. Vanilla Ossie rejects the key: it has no junction-table syntax and no custom_extensions encoding for one, so accepting it there would silently drop the junction on load. The serializer writes the block back, so a many-to-many model round-trips through a pull whole instead of collapsing to a direct-FK view. The two committed goldens (school_manytomany.{bigquery,spanner}.golden.sql, run against a live BigQuery instance and traversed with a GQL MATCH) now render from an authored document rather than hand-built IR, byte for byte -- so the format-to-DDL path is covered end to end. --- toolbox/mdcode/src/libts/semantic/ir.ts | 5 +- toolbox/mdcode/src/libts/semantic/loader.ts | 152 +++++++++++++++--- .../src/libts/semantic/osi_converter.ts | 53 +++--- .../src/libts/semantic/resolve_profiles.ts | 6 + .../tests/libts/semantic/bigquery.test.ts | 85 +++------- .../semantic/fixtures/school_manytomany.yaml | 65 ++++++++ .../tests/libts/semantic/loader.test.ts | 95 +++++++++++ .../libts/semantic/osi_converter.test.ts | 91 ++++++----- .../tests/libts/semantic/osi_schema.test.ts | 31 ++++ .../tests/libts/semantic/spanner.test.ts | 104 ++++-------- 10 files changed, 466 insertions(+), 221 deletions(-) create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.yaml diff --git a/toolbox/mdcode/src/libts/semantic/ir.ts b/toolbox/mdcode/src/libts/semantic/ir.ts index 5635f52c..73070876 100644 --- a/toolbox/mdcode/src/libts/semantic/ir.ts +++ b/toolbox/mdcode/src/libts/semantic/ir.ts @@ -250,8 +250,9 @@ export interface RelationshipEnd { * OWN key (`keys`) and may carry edge `fields` (properties of the association * itself, e.g. an enrollment's grade). Each side names the columns ON THE * JUNCTION TABLE that reference the corresponding endpoint entity's declared - * `keys`. The open format has no association-table syntax yet, so this is - * produced by hand-built IR (or a future format extension), not the loader. + * `keys`. Authored as the `association` block on a relationship, which is a + * native key of the extended profile ('0.2.0.dev0/google') only -- vanilla + * Ossie has no junction-table syntax. See loader.associationSchema. */ export interface Association { dataSource: string; // the junction table backing the edge diff --git a/toolbox/mdcode/src/libts/semantic/loader.ts b/toolbox/mdcode/src/libts/semantic/loader.ts index 03536171..5400d3d5 100644 --- a/toolbox/mdcode/src/libts/semantic/loader.ts +++ b/toolbox/mdcode/src/libts/semantic/loader.ts @@ -12,7 +12,7 @@ import * as yaml from 'yaml'; import * as z from 'zod'; -import {Action, ActionParameter, AiContext, CustomExtension, DATA_TYPES, Entity, Executor, Field, Metric, Relationship, SemanticModel,} from './ir'; +import {Action, ActionParameter, AiContext, Association, CustomExtension, DATA_TYPES, Entity, Executor, Field, Metric, Relationship, SemanticModel,} from './ir'; import {referencedEntityNames} from './sql_expr_utils'; export interface LoadOptions { @@ -150,10 +150,71 @@ const datasetBase = z.object({ custom_extensions: z.array(customExtensionSchema).optional(), }); +// The shape rules a relationship must satisfy under either version, shared by +// the base shape above and the strict per-load schemas in buildDocumentSchema +// so the two cannot drift. +// +// A direct foreign key binds the edge with the endpoints' own columns; a +// junction table binds it with the junction's. They are alternatives, so a +// relationship carries one set or the other, never both and never half of one. +function refineRelationship( + r: {name: string; from_columns?: string[]; to_columns?: string[]; + association?: unknown}, + ctx: z.RefinementCtx) { + if (r.association !== undefined && + (r.from_columns !== undefined || r.to_columns !== undefined)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `relationship '${r.name}': a many-to-many edge is bound by its ` + + `association's from_columns/to_columns (columns on the junction ` + + `table), so the relationship's own from_columns/to_columns must be ` + + `removed -- neither endpoint holds a foreign key.`, + }); + return; + } + if ((r.from_columns === undefined) !== (r.to_columns === undefined)) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `relationship '${ + r.name}': from_columns and to_columns must be given ` + + `together (both bind the edge) or both omitted (a logical edge); one ` + + `without the other is a half-bound join.`, + }); + } +} + +// The junction table backing a MANY-TO-MANY relationship (GOOGLE_VERSION +// only; see Association in ./ir). +// +// A many-to-many link cannot be a foreign key: an FK column holds one value and +// so references at most one row. The pairs live in a table of their own +// instead, one row per (from, to) -- an `enrollment` row per (student, course). +// That table is what `source` names. +// +// The `from_columns`/`to_columns` HERE are columns on the JUNCTION table, each +// referencing the corresponding endpoint entity's declared key. That is why the +// relationship's own `from_columns`/`to_columns` must be absent when this block +// is present: neither endpoint holds a foreign key. +const associationSchema = z.object({ + source: z.string(), + // The edge's own key on the junction table. Optional: when omitted the + // generators key the edge by its two endpoint column lists, deduplicated. + keys: z.array(z.string()).min(1).optional(), + from_columns: z.array(z.string()).min(1), + to_columns: z.array(z.string()).min(1), + // Properties of the pairing itself -- an enrollment's grade. Same shape as an + // entity's fields. + fields: z.array(fieldBase).optional(), +}); + const relationshipSchema = z.object({ name: z.string(), from: z.string(), to: z.string(), + // Present only on a many-to-many edge, which is + // backed by a junction table rather than by a + // foreign key (GOOGLE_VERSION only). + association: associationSchema.optional(), // Join columns are the physical binding of the // edge and are OPTIONAL, so a purely logical // relationship (an ontology edge, direction only) @@ -170,15 +231,7 @@ const relationshipSchema = z.object({ custom_extensions: z.array(customExtensionSchema).optional(), }).superRefine((r, ctx) => { - if ((r.from_columns === undefined) !== (r.to_columns === undefined)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `relationship '${ - r.name}': from_columns and to_columns must be given ` + - `together (both bind the edge) or both omitted (a logical edge); one ` + - `without the other is a half-bound join.`, - }); - } + refineRelationship(r, ctx); }); const metricSchema = z.object({ @@ -348,6 +401,27 @@ function buildDocumentSchema(bindingOptional: boolean, extended: boolean) { // either `bindingOptional`. }); + // A field inside an association is the same shape as an entity's, minus the + // extension carrier the version does not allow. + const associationField = z.object({ + name: z.string(), + expression: expressionSchema.optional(), + datatype: z.enum(DATA_TYPES).optional(), + description: z.string().optional(), + label: z.string().optional(), + dimension: dimensionSchema.optional(), + ai_context: aiContextSchema.optional(), + ...ce, + }).strict(); + + const association = z.object({ + source: z.string(), + keys: z.array(z.string()).min(1).optional(), + from_columns: z.array(z.string()).min(1), + to_columns: z.array(z.string()).min(1), + fields: z.array(associationField).optional(), + }).strict(); + const relationship = z.object({ name: z.string(), @@ -358,20 +432,15 @@ function buildDocumentSchema(bindingOptional: boolean, extended: boolean) { description: z.string().optional(), ai_context: aiContextSchema.optional(), ...ce, + // Many-to-many is a native extension: vanilla Ossie has no + // junction-table syntax and no `custom_extensions` encoding for one, + // so under OSSIE_VERSION the key is rejected as unknown rather than + // silently dropped. + ...(extended ? {association: association.optional()} : {}), }) .strict() .superRefine((r, ctx) => { - if ((r.from_columns === undefined) !== - (r.to_columns === undefined)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: - `relationship '${ - r.name}': from_columns and to_columns must be given ` + - `together (both bind the edge) or both omitted (a logical edge); one ` + - `without the other is a half-bound join.`, - }); - } + refineRelationship(r, ctx); }); const metric = @@ -449,6 +518,7 @@ type ExpressionDoc = z.infer; type DatasetDoc = z.infer; type FieldDoc = z.infer; type RelationshipDoc = z.infer; +type AssociationDoc = z.infer; type MetricDoc = z.infer; type ModelDoc = z.infer; type ActionDoc = z.infer; @@ -662,7 +732,10 @@ function convertModel( const entityNameSet = new Set(entityNames); const relationships = - (m.relationships ?? []).map(r => convertRelationship(r, entityNameSet)); + (m.relationships ?? []) + .map( + r => convertRelationship( + r, entityNameSet, opts, warnings, dialect)); rejectDuplicateNames( relationships.map(r => r.name), 'relationship name', `model '${m.name}'`); @@ -785,7 +858,8 @@ function convertField( // column arity) is a hard error, not a warning: the resulting edge would be // structurally invalid. function convertRelationship( - r: RelationshipDoc, entityNames: Set): Relationship { + r: RelationshipDoc, entityNames: Set, opts: LoadOptions, + warnings: string[], dialect: string): Relationship { const ctx = `relationship '${r.name}'`; if (!entityNames.has(r.from)) { throw new Error( @@ -810,6 +884,10 @@ function convertRelationship( source: {entity: r.from, columns: fromColumns}, destination: {entity: r.to, columns: toColumns}, }; + if (r.association) { + relationship.association = + convertAssociation(r.association, r.name, opts, warnings, dialect); + } const description = composeDescription(r.description); if (description) relationship.description = description; const ai = aiContextOrUndefined(r.ai_context); @@ -819,6 +897,34 @@ function convertRelationship( return relationship; } +// Maps the junction-table block of a many-to-many relationship onto the IR's +// Association. +// +// `keys` is the edge's own key. The format leaves it optional because the pair +// of endpoint column lists is already unique in the common case, so when it is +// omitted the key is those two lists concatenated and deduplicated -- the same +// default the BigQuery and Spanner generators would otherwise have to invent +// separately. Give it explicitly when the junction has a surrogate key, or when +// a pair may legitimately repeat (an enrollment per term). +function convertAssociation( + a: AssociationDoc, relName: string, opts: LoadOptions, warnings: string[], + dialect: string): Association { + const ctxLabel = `relationship '${relName}' association`; + const fields = + (a.fields ?? []).map(f => convertField(f, relName, warnings, dialect)); + rejectDuplicateNames(fields.map(f => f.name), 'field name', ctxLabel); + + const keys = a.keys ?? [...new Set([...a.from_columns, ...a.to_columns])]; + const association: Association = { + dataSource: parseSource(a.source, opts, warnings, ctxLabel), + keys, + sourceColumns: a.from_columns, + destinationColumns: a.to_columns, + }; + if (fields.length) association.fields = fields; + return association; +} + function convertMetric( mt: MetricDoc, entityNames: string[], warnings: string[], dialect: string): Metric { diff --git a/toolbox/mdcode/src/libts/semantic/osi_converter.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts index d504167f..ab351a62 100644 --- a/toolbox/mdcode/src/libts/semantic/osi_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -36,13 +36,13 @@ // any other vendor extension on the IR is dropped with a warning (its carrier's // fate under '/google' is still open). See serialize.test.ts. // -// An `association` (junction-table) relationship has no open-format syntax (the -// loader cannot produce one), so only its direct foreign-key view (from/to + -// columns) is serialized; the junction detail is dropped with a note. +// A many-to-many relationship round-trips whole: its `association` block is a +// native key of the extended profile, so the junction table, the edge's key, +// the junction-side columns and the edge properties are all written back. import * as yaml from 'yaml'; -import {Action, AiContext, CustomExtension, Entity, Executor, Field, Metric, Relationship, SemanticModel,} from './ir'; +import {Action, AiContext, Association, CustomExtension, Entity, Executor, Field, Metric, Relationship, SemanticModel,} from './ir'; // The version stamped on every serialized document. Pull emits kcmd's extended // profile: it uses native extension keys (`entities`, `deployment_target`) @@ -194,7 +194,8 @@ function modelDoc(model: SemanticModel, warnings: string[], logical: boolean): deployment_target: deploymentTarget, entities: datasets, relationships: nonEmpty( - (model.relationships ?? []).map(r => relationshipDoc(r, warnings))), + (model.relationships ?? []) + .map(r => relationshipDoc(r, warnings, logical))), metrics: nonEmpty((model.metrics ?? []).map(m => metricDoc(m, warnings))), actions: nonEmpty((model.actions ?? []).map(a => actionDoc(a, warnings))), @@ -308,30 +309,46 @@ function executorDoc(ex: Executor): Record { } // Inverts loader.convertRelationship: `from`/`to` are the endpoint entities and -// `from_columns`/`to_columns` are their positional join columns. An association -// (junction-table) edge has no open-format syntax, so only this direct-FK view -// is emitted and the junction detail is flagged. +// `from_columns`/`to_columns` are their positional join columns. A many-to-many +// edge instead carries an `association` block and, per the format, no join +// columns of its own -- the columns that bind it are on the junction table. function relationshipDoc( - rel: Relationship, warnings: string[]): Record { - if (rel.association) { - warnings.push( - `relationship '${ - rel.name}': association (junction-table) detail has no ` + - `open-format representation and is not serialized; only its foreign-key ` + - `endpoints are written.`); - } + rel: Relationship, warnings: string[], + logical: boolean): Record { dropExtensions(rel.customExtensions, `relationship '${rel.name}'`, warnings); + const association = rel.association ? + associationDoc(rel.association, rel.name, warnings, logical) : + undefined; return compact({ name: rel.name, from: rel.source.entity, to: rel.destination.entity, - from_columns: nonEmpty(rel.source.columns), - to_columns: nonEmpty(rel.destination.columns), + from_columns: association ? undefined : nonEmpty(rel.source.columns), + to_columns: association ? undefined : nonEmpty(rel.destination.columns), + association, description: rel.description, ai_context: aiContextDoc(rel.aiContext), }); } +// Inverts loader.convertAssociation. `keys` is always written even though the +// format lets it be omitted: the loader's default is derived from the two +// column lists, and re-deriving it on the way out would silently rewrite an +// edge whose authored key differed from that default. +function associationDoc( + assoc: Association, relName: string, warnings: string[], + logical: boolean): Record { + return compact({ + source: assoc.dataSource, + keys: nonEmpty(assoc.keys), + from_columns: nonEmpty(assoc.sourceColumns), + to_columns: nonEmpty(assoc.destinationColumns), + fields: nonEmpty( + (assoc.fields ?? []) + .map(f => fieldDoc(f, warnings, logical))), + }); +} + // Renders the `expression` object matching the loader's schema (a `dialects` // array of {dialect, expression}). Emits the target/canonical form under // CANONICAL_DIALECT and the imported vendor form under its own dialect, so the diff --git a/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts b/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts index 40fbf1a7..36043b5d 100644 --- a/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts +++ b/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts @@ -409,6 +409,12 @@ function firstUnboundReferenced( // The first join field of a relationship that is unbound on its own end, or // null when both ends' join columns are bound. +// +// A many-to-many edge has no columns on either end -- the columns that bind it +// are on its junction table, which a profile does not reach -- so it is never +// dropped here. It still falls with an endpoint: unbinding an entity's key +// makes that entity unavailable, and the loop above drops every edge touching +// it. function unboundJoinField(r: Relationship, unbound: Set): string|null { for (const c of r.source?.columns ?? []) { const key = `${r.source.entity}.${c}`; diff --git a/toolbox/mdcode/tests/libts/semantic/bigquery.test.ts b/toolbox/mdcode/tests/libts/semantic/bigquery.test.ts index 9e14a7c2..0c4648c7 100644 --- a/toolbox/mdcode/tests/libts/semantic/bigquery.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/bigquery.test.ts @@ -48,71 +48,26 @@ const GEN_OPTS: GenerateOptions = { }; -describe( - 'M:N association edge (no association-table syntax in the open format yet)', - () => { - // Hand-built because the loader's relationship schema is direct-FK only - // (from/to/columns) — it cannot express an edge backed by its own - // association table with its own KEY and edge properties. The expected - // DDL is a committed golden file - // (`school_manytomany.bigquery.golden.sql`) so the output stays - // reviewable as text; these exact strings were run against a live - // BigQuery instance and traversed with a GQL MATCH. - const SCHOOL: SemanticModel = { - name: 'school_graph', - entities: [ - { - name: 'students', - dataSource: 'sqlgen-testing.bei_semantic_ir_verify.students', - keys: ['student_id'], - fields: [ - {name: 'student_id', expression: 'students.student_id'}, - {name: 'name', expression: 'students.name'} - ] - }, - { - name: 'courses', - dataSource: 'sqlgen-testing.bei_semantic_ir_verify.courses', - keys: ['course_id'], - fields: [ - {name: 'course_id', expression: 'courses.course_id'}, - {name: 'title', expression: 'courses.title'} - ] - }, - ], - relationships: [ - { - name: 'enrollment', - source: {entity: 'students', columns: ['student_id']}, - destination: {entity: 'courses', columns: ['course_id']}, - association: { - dataSource: 'sqlgen-testing.bei_semantic_ir_verify.enrollment', - keys: ['enrollment_id'], - sourceColumns: ['student_id'], - destinationColumns: ['course_id'], - fields: [{ - name: 'grade', - expression: 'enrollment.grade', - description: 'Letter grade' - }] - } - }, - ], - metrics: [], - }; - const SCHOOL_OPTS: GenerateOptions = { - project: 'sqlgen-testing', - dataset: 'bei_semantic_ir_verify' - }; - - test('the association graph matches its committed golden DDL', () => { - const {ddl} = generatePropertyGraph(SCHOOL, SCHOOL_OPTS); - const golden = fs.readFileSync( - path.join(FIXTURES, 'school_manytomany.bigquery.golden.sql'), - 'utf8'); - expect(ddl).toBe(golden); - }); - }); +describe('M:N association edge', () => { + // Loaded from `school_manytomany.yaml`, which authors the junction table with + // the extended profile's `association` block, so this covers the whole path + // from the format to the DDL. The expected DDL is a committed golden file + // (`school_manytomany.bigquery.golden.sql`) so the output stays reviewable as + // text; these exact strings were run against a live BigQuery instance and + // traversed with a GQL MATCH. + const SCHOOL = loadFixture('school_manytomany.yaml'); + const SCHOOL_OPTS: GenerateOptions = { + project: 'sqlgen-testing', + dataset: 'bei_semantic_ir_verify' + }; + + test('the association graph matches its committed golden DDL', () => { + const {ddl} = generatePropertyGraph(SCHOOL, SCHOOL_OPTS); + const golden = fs.readFileSync( + path.join(FIXTURES, 'school_manytomany.bigquery.golden.sql'), 'utf8'); + expect(ddl).toBe(golden); + }); +}); describe('IR-contract metric cases the loader cannot produce', () => { diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.yaml new file mode 100644 index 00000000..905e61a0 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.yaml @@ -0,0 +1,65 @@ +# A many-to-many relationship: a student takes many courses and a course is +# taken by many students, so the pairs live in an `enrollment` table of their +# own rather than in a foreign key on either side. +# +# The `association` block is a native key of the extended profile, so this +# document declares `0.2.0.dev0/google`. Its BigQuery and Spanner DDL are the +# committed `school_manytomany.*.golden.sql` goldens, which were run against a +# live BigQuery instance and traversed with a GQL MATCH. + +version: "0.2.0.dev0/google" + +semantic_model: + - name: school_graph + description: Students, courses, and the enrollments that pair them + entities: + - name: students + source: sqlgen-testing.bei_semantic_ir_verify.students + primary_key: [student_id] + fields: + - name: student_id + expression: + dialects: + - dialect: BIGQUERY + expression: students.student_id + - name: name + expression: + dialects: + - dialect: BIGQUERY + expression: students.name + + - name: courses + source: sqlgen-testing.bei_semantic_ir_verify.courses + primary_key: [course_id] + fields: + - name: course_id + expression: + dialects: + - dialect: BIGQUERY + expression: courses.course_id + - name: title + expression: + dialects: + - dialect: BIGQUERY + expression: courses.title + + relationships: + - name: enrollment + from: students + to: courses + # No from_columns/to_columns on the relationship itself: neither student + # nor course holds a foreign key. The columns that bind the edge are on + # the junction table below. + association: + source: sqlgen-testing.bei_semantic_ir_verify.enrollment + keys: [enrollment_id] + from_columns: [student_id] + to_columns: [course_id] + # A property of the pairing, not of either endpoint. + fields: + - name: grade + expression: + dialects: + - dialect: BIGQUERY + expression: enrollment.grade + description: Letter grade diff --git a/toolbox/mdcode/tests/libts/semantic/loader.test.ts b/toolbox/mdcode/tests/libts/semantic/loader.test.ts index 6f3ea388..9cb50fb7 100644 --- a/toolbox/mdcode/tests/libts/semantic/loader.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/loader.test.ts @@ -290,6 +290,101 @@ describe('relationships map onto the direct-FK IR convention', () => { }); }); +describe('a many-to-many relationship is bound by its association', () => { + // Two datasets plus one relationship, so each case below only has to supply + // the relationship body under test. + const school = (relationship: object, version = '0.2.0.dev0/google') => + fromDocument({ version, + semantic_model: [{ + name: 'school', + datasets: [ + { name: 'students', source: 'students', primary_key: ['id'], fields: [] }, + { name: 'courses', source: 'courses', primary_key: ['id'], fields: [] }, + ], + relationships: [{ name: 'enrollment', from: 'students', to: 'courses', ...relationship }], + }], + }); + + const full = { + association: { + source: 'enrollment', + keys: ['enrollment_id'], + from_columns: ['student_id'], + to_columns: ['course_id'], + fields: [{ name: 'grade', expression: 'enrollment.grade' }], + }, + }; + + test('the junction table, its key, and the endpoint columns land on the IR', () => { + const rel = school(full).models[0].relationships[0]; + expect(rel.association).toEqual({ + // `source` is qualified like any other table binding. + dataSource: 'enrollment', + keys: ['enrollment_id'], + // from/to on the association are columns ON THE JUNCTION, so they map to + // the IR's sourceColumns/destinationColumns rather than to the endpoints. + sourceColumns: ['student_id'], + destinationColumns: ['course_id'], + fields: [{ name: 'grade', expression: 'enrollment.grade' }], + }); + }); + + test('neither endpoint carries join columns: no foreign key exists', () => { + const rel = school(full).models[0].relationships[0]; + expect(rel.source).toEqual({ entity: 'students', columns: [] }); + expect(rel.destination).toEqual({ entity: 'courses', columns: [] }); + }); + + test('an omitted key defaults to the two column lists, deduplicated', () => { + const rel = school({ + association: { + source: 'enrollment', + from_columns: ['student_id'], + to_columns: ['course_id'], + }, + }).models[0].relationships[0]; + expect(rel.association!.keys).toEqual(['student_id', 'course_id']); + }); + + test('a column shared by both endpoints appears once in the default key', () => { + const rel = school({ + association: { source: 'j', from_columns: ['a', 'b'], to_columns: ['b', 'c'] }, + }).models[0].relationships[0]; + expect(rel.association!.keys).toEqual(['a', 'b', 'c']); + }); + + test('the endpoint column lists need not be the same length', () => { + // They reference two different entities' keys, so a composite key on one + // side and a single column on the other is a valid junction -- unlike a + // direct foreign key, whose two lists pair up positionally. + const rel = school({ + association: { source: 'j', from_columns: ['a', 'b'], to_columns: ['c'] }, + }).models[0].relationships[0]; + expect(rel.association!.sourceColumns).toEqual(['a', 'b']); + expect(rel.association!.destinationColumns).toEqual(['c']); + }); + + test('an association alongside the relationship\'s own join columns is a hard error', () => { + // The two are alternative bindings. Accepting both would leave it undefined + // which one the graph is built from. + expect(() => school({ ...full, from_columns: ['id'], to_columns: ['id'] })) + .toThrow(/must be removed/); + }); + + test('an association with only one endpoint column list is a hard error', () => { + expect(() => school({ + association: { source: 'enrollment', from_columns: ['student_id'] }, + })).toThrow(); + }); + + test('vanilla Ossie rejects the association key', () => { + // Many-to-many is a native extension of the extended profile; vanilla + // Ossie has no junction-table syntax and no carrier for one, so the key is + // unknown rather than silently dropped. + expect(() => school(full, '0.2.0.dev0')).toThrow(/association/); + }); +}); + describe('abstract datasets and their source constraint', () => { test('a non-abstract dataset with no source is a hard error', () => { expect(() => fromDocument({ version: '0.2.0.dev0', diff --git a/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts index 25f43616..c0b1674e 100644 --- a/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts @@ -16,7 +16,7 @@ import * as path from 'node:path'; import * as yaml from 'yaml'; import {Field, Metric, Relationship, SemanticModel} from '../../../src/libts/semantic/ir'; -import {loadModels} from '../../../src/libts/semantic/loader'; +import {fromDocument, loadModels} from '../../../src/libts/semantic/loader'; import {modelDocument, serializeModel} from '../../../src/libts/semantic/osi_converter'; const FIXTURES = path.join(__dirname, 'fixtures'); @@ -187,49 +187,54 @@ describe('expression + datatype + dimension mapping', () => { }); -describe('lossy edges are flagged', () => { - test( - 'an association relationship warns and drops the junction detail', () => { - const rel: Relationship = { - name: 'enrollment', - source: {entity: 'student', columns: ['id']}, - destination: {entity: 'course', columns: ['id']}, - association: { - dataSource: 'p.d.enrollment', - keys: ['student_id', 'course_id'], - sourceColumns: ['student_id'], - destinationColumns: ['course_id'], - }, - }; - const model: SemanticModel = { - name: 'school', - entities: [ - { - name: 'student', - dataSource: 'p.d.student', - keys: ['id'], - fields: [] - }, - { - name: 'course', - dataSource: 'p.d.course', - keys: ['id'], - fields: [] - }, - ], - relationships: [rel], - metrics: [], - }; - const {yaml: text, warnings} = serializeModel(model); - expect(warnings.some(w => /association/i.test(w))).toBe(true); - // The direct-FK view is still emitted (from/to + columns), so it - // reloads. - const relDoc = yaml.parse(text).semantic_model[0].relationships[0]; - expect(relDoc.from).toBe('student'); - expect(relDoc.to).toBe('course'); - expect(relDoc.from_columns).toEqual(['id']); - }); +describe('many-to-many relationships', () => { + test('an association round-trips whole', () => { + const rel: Relationship = { + name: 'enrollment', + source: {entity: 'student', columns: []}, + destination: {entity: 'course', columns: []}, + association: { + dataSource: 'p.d.enrollment', + keys: ['student_id', 'course_id'], + sourceColumns: ['student_id'], + destinationColumns: ['course_id'], + fields: [{name: 'grade', expression: 'grade', type: 'String'}], + }, + }; + const model: SemanticModel = { + name: 'school', + entities: [ + {name: 'student', dataSource: 'p.d.student', keys: ['id'], fields: []}, + {name: 'course', dataSource: 'p.d.course', keys: ['id'], fields: []}, + ], + relationships: [rel], + metrics: [], + }; + const {yaml: text, warnings} = serializeModel(model); + expect(warnings.some(w => /association/i.test(w))).toBe(false); + + // The junction detail is a native key now, so it survives serialization + // instead of collapsing to a direct-FK view. + const relDoc = yaml.parse(text).semantic_model[0].relationships[0]; + expect(relDoc.from).toBe('student'); + expect(relDoc.to).toBe('course'); + // A many-to-many edge carries no join columns of its own; the columns that + // bind it are on the junction table. + expect(relDoc.from_columns).toBeUndefined(); + expect(relDoc.to_columns).toBeUndefined(); + expect(relDoc.association.source).toBe('p.d.enrollment'); + expect(relDoc.association.keys).toEqual(['student_id', 'course_id']); + expect(relDoc.association.from_columns).toEqual(['student_id']); + expect(relDoc.association.to_columns).toEqual(['course_id']); + expect(relDoc.association.fields[0].name).toBe('grade'); + // And it reloads into the same IR. + const reloaded = fromDocument(yaml.parse(text)).models[0]; + expect(reloaded.relationships[0].association).toEqual(rel.association!); + }); +}); + +describe('lossy edges are flagged', () => { test('a non-GOOGLE vendor extension is dropped with a warning', () => { // The extended ('/google') profile has no custom_extensions carrier, so a // non-deployment-target vendor extension has no representation and is diff --git a/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts index 5dc645b4..83d51c47 100644 --- a/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts @@ -159,6 +159,31 @@ function onlyActionsExtension(errors: typeof validate.errors): boolean { /\/semantic_model\/\d+$/.test(e.instancePath)); } +// A many-to-many `association` block is the other deliberate SUPERSET of +// released Apache OSI. The released schema knows only the direct foreign-key +// edge, so a junction-backed relationship trips it twice: `association` is an +// additional property, and the `from_columns`/`to_columns` it required are +// absent -- correctly, because a many-to-many edge has none of its own. We +// tolerate EXACTLY those three errors on a /relationships/ path and nothing +// else. When upstream OSI adopts a junction-table syntax, re-vendoring the +// schema makes this pass with no special-casing. +function onlyAssociationExtension(errors: typeof validate.errors): boolean { + const missingOk = new Set(['from_columns', 'to_columns']); + return !!errors && errors.length > 0 && + errors.every(e => { + if (!/\/relationships\/\d+$/.test(e.instancePath)) return false; + if (e.keyword === 'required') { + return missingOk.has( + (e.params as {missingProperty?: string}).missingProperty ?? ''); + } + if (e.keyword === 'additionalProperties') { + return (e.params as {additionalProperty?: string}) + .additionalProperty === 'association'; + } + return false; + }); +} + describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => { test('at least one fixture is discovered', () => { expect(fixtures.length).toBeGreaterThan(0); @@ -201,6 +226,12 @@ describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => onlyActionsExtension(validate.errors)) { return; } + // A junction-backed relationship is a deliberate superset too; tolerate + // exactly its three errors, and only on the fixture that carries one. + if (rel === 'school_manytomany.yaml' && + onlyAssociationExtension(validate.errors)) { + return; + } const details = (validate.errors ?? []) .map(e => ` ${e.instancePath || '(root)'} ${e.message}`).join('\n'); throw new Error(`OSI schema validation failed for ${rel}:\n${details}`); diff --git a/toolbox/mdcode/tests/libts/semantic/spanner.test.ts b/toolbox/mdcode/tests/libts/semantic/spanner.test.ts index af0a1693..3aff9896 100644 --- a/toolbox/mdcode/tests/libts/semantic/spanner.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/spanner.test.ts @@ -16,84 +16,48 @@ import * as fs from 'fs'; import * as path from 'path'; import {SemanticModel} from '../../../src/libts/semantic/ir'; +import {loadModels} from '../../../src/libts/semantic/loader'; import {GenerateOptions, generateSpannerPropertyGraph} from '../../../src/libts/semantic/spanner'; const FIXTURES = path.join(__dirname, 'fixtures'); -describe( - 'M:N association edge (no association-table syntax in the open format yet)', - () => { - // Hand-built because the loader's relationship schema is direct-FK only; - // it cannot express an edge backed by its own association table with its - // own KEY and edge properties. The expected DDL is a committed golden - // (`school_manytomany.spanner.golden.sql`), the Spanner counterpart to - // the BigQuery association golden, so the two shapes are reviewable side - // by side. - const SCHOOL: SemanticModel = { - name: 'school_graph', - entities: [ - { - name: 'students', - dataSource: 'sqlgen-testing.bei_semantic_ir_verify.students', - keys: ['student_id'], - fields: [ - {name: 'student_id', expression: 'students.student_id'}, - {name: 'name', expression: 'students.name'} - ] - }, - { - name: 'courses', - dataSource: 'sqlgen-testing.bei_semantic_ir_verify.courses', - keys: ['course_id'], - fields: [ - {name: 'course_id', expression: 'courses.course_id'}, - {name: 'title', expression: 'courses.title'} - ] - }, - ], - relationships: [ - { - name: 'enrollment', - source: {entity: 'students', columns: ['student_id']}, - destination: {entity: 'courses', columns: ['course_id']}, - association: { - dataSource: 'sqlgen-testing.bei_semantic_ir_verify.enrollment', - keys: ['enrollment_id'], - sourceColumns: ['student_id'], - destinationColumns: ['course_id'], - fields: [{ - name: 'grade', - expression: 'enrollment.grade', - description: 'Letter grade' - }] - } - }, - ], - metrics: [], - }; +// Loads a fixture to its IR, the way the BigQuery suite does. +function loadFixture(fixture: string): SemanticModel { + const text = fs.readFileSync(path.join(FIXTURES, fixture), 'utf8'); + const {models} = loadModels( + text, {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}); + return models[0]; +} + +describe('M:N association edge', () => { + // Loaded from `school_manytomany.yaml`, the same document the BigQuery suite + // renders, so one authored many-to-many model is shown deploying to either + // store. The expected DDL is a committed golden + // (`school_manytomany.spanner.golden.sql`), the Spanner counterpart to the + // BigQuery association golden, so the two shapes are reviewable side by side. + const SCHOOL = loadFixture('school_manytomany.yaml'); + + test('the association graph matches its committed golden DDL', () => { + const {ddl} = generateSpannerPropertyGraph(SCHOOL); + const golden = path.join(FIXTURES, 'school_manytomany.spanner.golden.sql'); + if (process.env.UPDATE_GOLDENS) { + fs.writeFileSync(golden, ddl); + return; + } + expect(ddl).toBe(fs.readFileSync(golden, 'utf8')); + }); - test('the association graph matches its committed golden DDL', () => { + test( + 'an edge property carries no OPTIONS (Spanner has no per-element options)', + () => { + // The junction's `grade` field has a description; on BigQuery that + // becomes an OPTIONS clause, on Spanner it is dropped. const {ddl} = generateSpannerPropertyGraph(SCHOOL); - const golden = - path.join(FIXTURES, 'school_manytomany.spanner.golden.sql'); - if (process.env.UPDATE_GOLDENS) { - fs.writeFileSync(golden, ddl); - return; - } - expect(ddl).toBe(fs.readFileSync(golden, 'utf8')); + expect(ddl).toContain('grade'); + expect(ddl).not.toContain('OPTIONS'); }); - - test( - 'an edge property carries no OPTIONS (Spanner has no per-element options)', - () => { - // The junction's `grade` field has a description; on BigQuery that - // becomes an OPTIONS clause, on Spanner it is dropped. - const {ddl} = generateSpannerPropertyGraph(SCHOOL); - expect(ddl).toContain('grade'); - expect(ddl).not.toContain('OPTIONS'); - }); - }); +}); describe('graph naming', () => { From f25b94bd765cbd7943630c8894d4cbd41f988f91 Mon Sep 17 00:00:00 2001 From: Bei Li Date: Tue, 8 Sep 2026 16:25:40 +0000 Subject: [PATCH 2/3] mdcode: publish many-to-many relationships to Knowledge Catalog A `schema-join` entry link holds exactly one source/target column pair, so it cannot describe an edge that runs through a junction table, and Dataplex has no custom entry-LINK types -- only custom entry and aspect types. So a many-to-many relationship is published as an entry instead: one `semantic-association` entry per relationship, parented to the model entry, holding the two entities it pairs and the junction table with its keys, join columns, and fields. The custom type is added the same way `semantic-action` was: one entry type and one aspect type sharing an id, appended to CUSTOM_TYPES in kc_custom_types.ts and provisioned by `kcmd init --semantic-model` in the destination project at global. Nothing in provisioning, naming, or init wiring needed changing. The encoding lives in kc_associations.ts, mirroring kc_actions.ts. Both the relationship's `instructions` and the edge's own fields ride the custom aspect rather than the built-in `guidelines` / `schema` aspects, because a pull derives the aspect base from the entry type's project and the custom type lives in the destination project. Field expressions are stored unconditionally -- the `--emit-expressions` gate exists for the published system templates that lack the fields; this template is ours. Push, pull, and the round-trip goldens cover the new shape, and the docs (model spec, fidelity, reference, README) describe it alongside the foreign-key edge. --- toolbox/mdcode/docs/semantic-model/README.md | 8 + .../mdcode/docs/semantic-model/fidelity.md | 48 ++- .../mdcode/docs/semantic-model/model_spec.md | 67 +++- .../mdcode/docs/semantic-model/reference.md | 48 ++- .../src/libts/semantic/kc_associations.ts | 354 ++++++++++++++++++ .../mdcode/src/libts/semantic/kc_converter.ts | 43 ++- .../src/libts/semantic/kc_custom_types.ts | 180 ++++++++- .../src/libts/semantic/knowledge_catalog.ts | 81 ++-- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 7 +- ...l_manytomany.knowledge_catalog.golden.json | 139 +++++++ .../school_manytomany.osi.golden.yaml | 54 +++ .../school_manytomany.pull.golden.yaml | 44 +++ .../tests/libts/semantic/kc_converter.test.ts | 152 +++++++- .../semantic/knowledge_catalog.e2e.test.ts | 4 +- .../libts/semantic/knowledge_catalog.test.ts | 188 +++++++++- .../libts/semantic/osi_converter.test.ts | 1 + .../tests/libts/semantic/osi_schema.test.ts | 61 +-- .../tests/tool/init_semantic_model.test.ts | 103 +++-- 18 files changed, 1396 insertions(+), 186 deletions(-) create mode 100644 toolbox/mdcode/src/libts/semantic/kc_associations.ts create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.knowledge_catalog.golden.json create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.osi.golden.yaml create mode 100644 toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.pull.golden.yaml diff --git a/toolbox/mdcode/docs/semantic-model/README.md b/toolbox/mdcode/docs/semantic-model/README.md index d6bd90e0..393c9447 100644 --- a/toolbox/mdcode/docs/semantic-model/README.md +++ b/toolbox/mdcode/docs/semantic-model/README.md @@ -114,6 +114,14 @@ Field and relationship names are the business vocabulary — `order_id`, metric's `expression` may be a bare formula over the logical fields or the fuller per-dialect form. `entities` may also be written `datasets` (the two are interchangeable under the `/google` version). +A relationship that pairs many rows on each side — a student takes many courses, +a course has many students — is written with an `association` block instead of +`from_columns` / `to_columns`. The block names the junction table that holds the +pairs, the columns that reach each side, and any fields the pairing itself +carries (an enrollment's grade, say). Both graphs deploy it as an edge table over +the junction; Knowledge Catalog stores it as a `semantic-association` entry. See +[Model spec §2.2.1](model_spec.md#221-many-to-many-association). + Entities can **extend** other entities (`extends: [Parent]`); push flattens the supertype's fields down and expresses the hierarchy as graph labels, so a query against the supertype gathers every subtype. See diff --git a/toolbox/mdcode/docs/semantic-model/fidelity.md b/toolbox/mdcode/docs/semantic-model/fidelity.md index f7652a4f..35e752bf 100644 --- a/toolbox/mdcode/docs/semantic-model/fidelity.md +++ b/toolbox/mdcode/docs/semantic-model/fidelity.md @@ -33,7 +33,7 @@ agree on every structural row and differ only where a Spanner target has no | Unique keys | `schema.uniqueConstraints` | ✓ | — dropped (only PK emitted) | — dropped (only PK emitted) | | Metric | `semantic-metric` entry | name, entity, description, instructions, type⁵ | `MEASURE`⁴ | — dropped (no `MEASURE`) | | Relationship (1:1 / 1:N) | `schema-join` link | ✓ (name normalized⁶) | `EDGE TABLE` | `EDGE TABLE` | -| Relationship (M:N / `association`) | — not stored | — | `EDGE TABLE` (via junction table) | `EDGE TABLE` (via junction table) | +| Relationship (M:N / `association`) | `semantic-association` entry¹³ | ✓¹³ | `EDGE TABLE` (via junction table) | `EDGE TABLE` (via junction table) | | Entity `extends` | — not modelled | — | `LABEL` clauses + flattened fields | `LABEL` clauses + flattened fields | | Action | `semantic-action` entry¹² | ✓¹² | — not represented (write-side) | — not represented (write-side) | | `description` (entity / metric / field / relationship) | entry description / aspect | ✓ | `OPTIONS(description)` | — dropped | @@ -63,14 +63,18 @@ agree on every structural row and differ only where a Spanner target has no 5. **Metric type.** A metric's expression is gated behind `--emit-expressions`; its data type round-trips only for a concrete type (e.g. `Decimal`) — an untyped, `String`, or `Opaque` metric comes back un-typed. -6. **Relationship name.** Relationship names come back lowercased/hyphenated - (`Places Order` → `places-order`) — the catalog stores the name only in the - link id. See [Writer-side follow-up](#writer-side-follow-up). +6. **Relationship name.** A one-to-many relationship's name comes back + lowercased/hyphenated (`Places Order` → `places-order`) — the catalog stores + the name only in the link id. See + [Writer-side follow-up](#writer-side-follow-up). A many-to-many relationship + is stored as an entry, not a link, so its name comes back verbatim. 7. **Guidelines aspect.** The `guidelines` aspect exists only for the model, entities, and metrics — not fields or relationships, so field- and relationship-level `ai_context.instructions` has no Knowledge Catalog home (a relationship's instructions still reach BigQuery, folded into the edge's - `OPTIONS(description)`). + `OPTIONS(description)`). The one exception is a many-to-many relationship, + whose `semantic-association` aspect carries its `instructions` — that aspect + type is ours, so it has a field for them. 8. **Model-level metadata.** Neither graph has a home for statement-level metadata — BigQuery silently drops graph-statement `OPTIONS`, and Spanner carries no `OPTIONS` at all — so the model's `description` and @@ -103,12 +107,23 @@ agree on every structural row and differ only where a Spanner target has no scope: an action's `precondition` and `affects` are not modelled, so nothing about them is stored either way. See [Modeling write operations](actions.md). +13. **Many-to-many relationships.** The `schema-join` link holds exactly one + source/target column pair, so it cannot describe an edge that runs through a + junction table. A many-to-many relationship is published instead as one + `semantic-association` entry under the model entry, holding the two entities + it pairs, the junction table, and the junction's keys, join columns, and + fields. The whole `association` block round-trips — including the edge's own + fields and their expressions, which are stored unconditionally (the aspect + type is ours, so it has fields for them; the `--emit-expressions` gate exists + for the published system templates that do not). See + [Model spec §2.2.1](model_spec.md#221-many-to-many-association). ## To Knowledge Catalog The catalog holds metadata rather than a full copy of your model. Every resource type it uses is a built-in system type under `dataplex-types/global`, apart from -the custom `semantic-action` pair that `kcmd init` provisions — push references +the custom `semantic-association` and `semantic-action` pairs that `kcmd init` +provisions — push references types, it never creates them (see [Reference → What gets created in Knowledge Catalog](reference.md#what-gets-created-in-knowledge-catalog)). @@ -135,6 +150,16 @@ SQL (`importedExpression` — for example the MAQL or Snowflake form a metric wa imported from). Those stay in your authored document; the vendor SQL and expressions are still used when generating graph SQL. +**Many-to-many relationships** get an entry rather than a link. A `schema-join` +link holds one source/target column pair, which cannot describe an edge that +runs through a junction table, and Knowledge Catalog has no custom *link* types +— only custom entry and aspect types. So each many-to-many relationship becomes +a `semantic-association` entry under the model entry, carrying the two entities +it pairs and the junction table with its keys, join columns, and fields. The +whole block round-trips through `pull`, name included. The entry type is custom, +so `kcmd init` creates it; a model with no many-to-many relationship never needs +it. + **Actions** follow the same one-entry-per-element rule as everything else: each becomes a `semantic-action` entry under the model entry, carrying its executor and typed parameters in a `semantic-action` aspect. They round-trip losslessly @@ -205,9 +230,10 @@ returns that view. Two things about *how* it comes back: **Normalized** — the content survives, the form changes: -- Relationship *names* come back lowercased/hyphenated (`Places Order` → - `places-order`); the catalog stores the name only in the link id. See - [Writer-side follow-up](#writer-side-follow-up). +- A one-to-many relationship's *name* comes back lowercased/hyphenated + (`Places Order` → `places-order`); the catalog stores the name only in the link + id. See [Writer-side follow-up](#writer-side-follow-up). A many-to-many + relationship is stored as an entry instead, so its name comes back verbatim. - Field types round-trip except two collapses: a field authored with no type comes back as `Opaque`, and a field authored as `String` comes back un-typed (both store `dataType STRING`, kept distinct by a field's `metadataType` — see @@ -232,8 +258,8 @@ One reduction above is a limit of what push currently *writes* rather than of what pull can recover. It is recorded here as a write-side follow-up; the reader (pull) already returns everything the catalog holds. -- **Relationship names.** The `schema-join` aspect type's `metadataTemplate` has - no field for the relationship name, so push cannot store it and pull recovers +- **Relationship names** (one-to-many only). The `schema-join` aspect type's + `metadataTemplate` has no field for the relationship name, so push cannot store it and pull recovers it from the link id — which is lowercased and hyphenated (the entry-link id format forbids the original casing/underscores). Returning the name verbatim requires adding a name field to the built-in `schema-join` aspect type in diff --git a/toolbox/mdcode/docs/semantic-model/model_spec.md b/toolbox/mdcode/docs/semantic-model/model_spec.md index 64719b25..13e56812 100644 --- a/toolbox/mdcode/docs/semantic-model/model_spec.md +++ b/toolbox/mdcode/docs/semantic-model/model_spec.md @@ -201,6 +201,7 @@ A relationship is a directed edge between two datasets. | `to` | string | required; a declared dataset name | | `from_columns` | list of strings | join key on `from` | | `to_columns` | list of strings | join key on `to` | +| `association` | [association](#221-many-to-many-association) | `/google` only; a many-to-many edge | | `description` | string | | | `ai_context` | [ai_context](#24-ai_context) | | | `custom_extensions` | list | [§6](#6-the-extension-mechanism) | @@ -217,10 +218,45 @@ requires them (see [§4](#4-narrowings-and-relaxations)). Unlike `source` and fi join columns are declared on the logical model and are not profile-swappable ([§7](#7-the-binding-layer)). -> **Many-to-many is not yet authorable.** A junction-table (M:N) relationship -> exists in `kcmd`'s internal representation but has **no YAML syntax** in -> `0.2.0.dev0`. It cannot be authored today and is reserved for a future format -> extension. Direct-FK relationships (1:1, 1:N) are the authorable forms. +#### 2.2.1. Many-to-many (`association`) + +A relationship whose two sides each match many of the other cannot be a foreign +key: a foreign-key column holds one value, so it references at most one row. The +pairs live in a table of their own — one row per pair — and the edge is declared +with an `association` block instead of the relationship's own join columns. + +| Key | Type | Rule | +|---|---|---| +| `source` | string | required; the table holding the pairs | +| `from_columns` | list of strings | required; columns on that table referencing `from` | +| `to_columns` | list of strings | required; columns on that table referencing `to` | +| `keys` | list of strings | the pairing's own key; defaults to the two column lists combined | +| `fields` | list of [field](#211-field) | properties of the pairing itself | + +```yaml +relationships: + - name: enrollment + from: students + to: courses + association: + source: analytics.school.enrollment + keys: [enrollment_id] + from_columns: [student_id] + to_columns: [course_id] + fields: + - name: grade + expression: enrollment.grade +``` + +The columns named inside the block are columns of the **pairing table**, not of +either endpoint, and each list references the corresponding endpoint's declared +`primary_key`. A relationship MUST NOT carry both an `association` and its own +`from_columns`/`to_columns`: an edge is one shape or the other. `fields` are +properties of the pairing rather than of either side — a grade belongs to the +enrollment, not to the student or the course. + +`association` is a native key of the extended profile +([§5](#5-extensions)); vanilla Ossie has no syntax for a pairing table. ### 2.3. Metric @@ -305,7 +341,7 @@ extension, [§5](#5-extensions)), or *rejected* / *not authorable* (excluded). | `abstract` | — | added | supertype with no table; `/google` only · [§5](#5-extensions) | | relationship `name`, `from`, `to` | defined | same | [§2.2](#22-relationship) | | relationship `from_columns` / `to_columns` | required | optional | model before binding; none = logical edge · [§4.2](#42-relaxations-looser-than-ossie) | -| relationship M:N (`association`) | — | not authorable (reserved) | no M:N syntax yet · [§2.2](#22-relationship) | +| relationship M:N (`association`) | — | added | pairing table + its own key and fields; `/google` only · [§2.2.1](#221-many-to-many-association), [§5](#5-extensions) | | `metrics`, metric `expression` | required | same; graph-bound stricter | a graph measure binds one node and aggregate · [§4.1](#41-narrowings-stricter-than-ossie) | | `expression.dialects` | closed enum | any dialect string | tolerate imported / newer input · [§4.2](#42-relaxations-looser-than-ossie) | | field `expression` (column binding) | required | optional | model before binding; unbound is pruned · [§4.2](#42-relaxations-looser-than-ossie), [§7](#7-the-binding-layer) | @@ -353,8 +389,9 @@ Each rule and its reason: measure. - **A graph-bound relationship MUST have its join columns bound.** For any graph - target, a non-M:N relationship MUST supply both `from_columns` and `to_columns` - before deploy. *Why:* the edge table needs both keys. + target, a relationship MUST supply both `from_columns` and `to_columns` before + deploy — on the relationship itself, or, for a many-to-many edge, inside its + `association` block. *Why:* the edge table needs both keys. - **Unknown keys are rejected.** Every object is validated closed: an unrecognized sibling key is a hard load error, not silently dropped. Combined with the version @@ -453,10 +490,11 @@ reads the document ([§6](#6-the-extension-mechanism)). bindings, so one logical model serves several stores. Not part of the Ossie document; a `kcmd`-specific file alongside it ([§7](#7-the-binding-layer)). -Deliberately **not** extensions in `0.2.0.dev0`, to avoid the impression they -exist: there is **no `actions` block** and **no authorable M:N `association` -syntax**. Both are reserved for future consideration; neither is part of the -format today. +- **`association` (extended profile only).** A many-to-many relationship, backed + by a table of pairs with its own key and its own properties. Ossie's + relationship is a foreign key only, and the carrier cannot express one either: + a `custom_extensions` block holds opaque data, and this edge has to be read by + the graph generators. Grammar in [§2.2.1](#221-many-to-many-association). ## 6. The extension mechanism @@ -605,9 +643,10 @@ The full merge behavior and worked examples are in and constructs with no vanilla form (inheritance, the `entities` spelling) are simply unavailable there — a model that needs them uses `0.2.0.dev0/google`. -- **Reserved constructs.** `association` (M:N) and any `actions`-like write-side - construct are reserved: recognized as future work, not authorable today. A - document MUST NOT rely on either in `0.2.0.dev0`. +- **Extensions are additive.** `association` and `actions` were both added to + the extended profile after `0.2.0.dev0/google` was first published, each as a + new optional key. A document that used neither is unaffected, which is the + shape any further extension takes. ## Appendix: annotated example diff --git a/toolbox/mdcode/docs/semantic-model/reference.md b/toolbox/mdcode/docs/semantic-model/reference.md index 07b4cd68..f3b37fc9 100644 --- a/toolbox/mdcode/docs/semantic-model/reference.md +++ b/toolbox/mdcode/docs/semantic-model/reference.md @@ -259,10 +259,10 @@ are **not** probed before deploy — the live pre-flight is BigQuery-only (see ## What gets created in Knowledge Catalog Each element of your model maps to one catalog resource. Every resource type -below except `semantic-action` is a built-in system type under -`dataplex-types/global` — push references them, it never creates them. -`semantic-action` is custom, and `kcmd init --semantic-model` creates it in your -own project at `global`; push still writes only entries. +below except `semantic-association` and `semantic-action` is a built-in system +type under `dataplex-types/global` — push references them, it never creates +them. Those two are custom, and `kcmd init --semantic-model` creates them in +your own project at `global`; push still writes only entries. > Set `KC_TYPE_PROJECT` to read these system types from another project, and > `DATAPLEX_ENDPOINT` to target a non-prod Dataplex host; both default to @@ -273,7 +273,8 @@ own project at `global`; push still writes only entries. | Model | `semantic-model` | entry — anchor / parent of the rest | `` | | Entity | `semantic-entity` (+ built-in `schema` aspect) | entry | `.entities.` | | Metric | `semantic-metric` | entry | `.metrics.` | -| Relationship | `schema-join` | entry link between the two entity entries | derived from the model and relationship names | +| Relationship (1:1 / 1:N) | `schema-join` | entry link between the two entity entries | derived from the model and relationship names | +| Relationship (M:N) | `semantic-association` (custom type) | entry | `.associations.` | | Action | `semantic-action` (custom type) | entry | `.actions.` | An entity entry carries its columns in the `schema` aspect (name, data type, @@ -283,6 +284,16 @@ relationship detail — the paired columns and foreign-key direction — in its aspect. Any element with `ai_context.instructions` (the model, an entity, or a metric) also gets a built-in `guidelines` aspect holding that text. +A **many-to-many** relationship gets an entry rather than a link: a +`schema-join` link holds exactly one source/target column pair, so it cannot +describe an edge that runs through a junction table, and Dataplex has no custom +*link* types — only custom entry and aspect types. Its `semantic-association` +aspect holds the two entities it pairs, the junction table, and the junction's +keys, join columns, and fields, along with the relationship's +`ai_context.instructions`. The whole `association` block round-trips through +`pull`, the relationship name included. See +[Model spec §2.2.1](model_spec.md#221-many-to-many-association). + An **action** entry carries its executor and its typed parameters in a `semantic-action` aspect, along with the action's `ai_context.instructions`. That aspect type is provisioned in your project rather than referenced from @@ -402,9 +413,10 @@ and each aspect type attached, so a push needs, on the destination entry group: `semantic-entity`, and `semantic-metric` aspect types the push attaches — i.e. `dataplex.entryGroups.useSemanticModelAspect`, `useSemanticEntityAspect`, and `useSemanticMetricAspect` -* `dataplex.aspectTypes.use` on the `semantic-action` aspect type, when the - model declares actions — that type is custom rather than built-in, so it is - authorized on the type resource instead of through an entry-group +* `dataplex.aspectTypes.use` on the `semantic-association` aspect type, when the + model has many-to-many relationships, and on the `semantic-action` aspect + type, when it declares actions — those types are custom rather than built-in, + so they are authorized on the type resource instead of through an entry-group use-permission > The `schema` / `guidelines` / `schema-join` use-permissions follow Dataplex's @@ -419,16 +431,16 @@ needs more than push does, in the destination project: * `dataplex.entryGroups.create` — the destination entry group * `dataplex.aspectTypes.create` / `dataplex.aspectTypes.update` and - `dataplex.entryTypes.create` — the custom `semantic-action` pair. Init patches - an aspect type that is already there, so a project set up by an older `kcmd` - picks up template additions; an entry type that is already there is left - alone. - -Only the entry-group permission is required. Actions are one optional -construct, so init reports a refusal to create their types as a warning and -carries on; every model that declares no action still pushes and pulls. Any -other failure to create a type stops init, rather than leaving a later push to -hit an opaque parsing error. + `dataplex.entryTypes.create` — the custom `semantic-association` and + `semantic-action` pairs. Init patches an aspect type that is already there, so + a project set up by an older `kcmd` picks up template additions; an entry type + that is already there is left alone. + +Only the entry-group permission is required. Many-to-many relationships and +actions are both optional constructs, so init reports a refusal to create their +types as a warning and carries on; a model that uses neither still pushes and +pulls. Any other failure to create a type stops init, rather than leaving a +later push to hit an opaque parsing error. `kcmd pull` needs read access to the same entry group instead — to list its entries and fetch each `semantic-*` entry with its aspects. diff --git a/toolbox/mdcode/src/libts/semantic/kc_associations.ts b/toolbox/mdcode/src/libts/semantic/kc_associations.ts new file mode 100644 index 00000000..f5172ea3 --- /dev/null +++ b/toolbox/mdcode/src/libts/semantic/kc_associations.ts @@ -0,0 +1,354 @@ +// How a model's MANY-TO-MANY relationships are encoded in Knowledge Catalog. +// +// A one-to-many relationship publishes as a built-in `schema-join` entry link +// between the two entity entries. A many-to-many one cannot: schema-join holds +// a single source/target column pair, and a junction is two joins through a +// third table. A custom entry LINK type is not an option either -- Dataplex +// accepts only its own link types. So a many-to-many edge publishes as an ENTRY +// instead, one per relationship, parented to the model anchor beside the +// entities and metrics, carrying an aspect that holds both joins and the +// junction table. The type is the custom `semantic-association` pair DECLARED +// IN kc_custom_types.ts and created there by `kcmd init --semantic-model`. That +// file is the list of what is custom; this one is only the encoding that fills +// the aspect. +// +// The custom pair is what makes a many-to-many edge explicit in the catalog. A +// search can tell one apart from anything else by its entry type, and the +// aspect's fields are typed and queryable rather than prose a reader has to +// interpret. +// +// WHEN A BUILT-IN MANY-TO-MANY TYPE SHIPS, follow the instructions at the top +// of kc_custom_types.ts. Nothing in this file changes: the readers below match +// a type by its id suffix, so they do not care which project it lives in. If +// what ships instead is a schema-join that can model a junction, this file goes +// away and `relationshipLink` in knowledge_catalog.ts stops skipping M:N. +// +// The call sites are `knowledge_catalog.ts` (emit the entries), +// `kc_converter.ts` (read them back), and `pull_kc.ts` (hydrate the aspect). +// +// `instructions` and the edge's own `fields` ride the association's aspect +// rather than the built-in `guidelines` and `schema` aspects an entity uses. A +// pull derives which aspect types to hydrate from the project the ENTRY type +// lives in, so an association entry, whose type is custom and therefore in the +// destination project, would ask for aspect types that exist only under +// `dataplex-types`. Keeping both on the association's own aspect keeps the +// whole encoding inside the one type kc_custom_types.ts provisions. +// +// The helpers at the bottom duplicate a few lines from the modules above on +// purpose. This module imports only the IR, the entry shape and the type +// registry, so it can be swapped or deleted as a unit. + +import {Entry} from '../gcp/dataplex'; + +import {AiContext, Association, DATA_TYPES, DataType, Field, Relationship, SemanticModel} from './ir'; +import {ASSOCIATION_TYPE_ID, customAspectKey, customAspectTypeName, customEntryTypeName} from './kc_custom_types'; + +// Full resource name of the association entry type for a destination. +export function associationEntryTypeName(dest: {project: string}): string { + return customEntryTypeName(ASSOCIATION_TYPE_ID, dest); +} + +// Full resource name of the association aspect type for a destination. +export function associationAspectTypeName(dest: {project: string}): string { + return customAspectTypeName(ASSOCIATION_TYPE_ID, dest); +} + +// Aspect-map key: the `project.location.type` reference form the client keys an +// entry's aspects by. +export function associationAspectKey(dest: {project: string}): string { + return customAspectKey(ASSOCIATION_TYPE_ID, dest); +} + + +// --------------------------------------------------------------------------- +// Write side: the IR -> one entry per many-to-many relationship. +// --------------------------------------------------------------------------- + +// What the emitter supplies so this file need not rebuild entry names or repeat +// the id-collision bookkeeping it already does for entities and metrics. +export interface AssociationEmitContext { + // The destination project, which is where the custom types live. + project: string; + // Full entry resource name for an entry id (Namer.entry). + entry(entryId: string): string; + // Full entry resource name of the model anchor, the parent of every + // association. + anchor: string; + // Reserves an entry id, returning false when it collides with one already + // emitted (knowledge_catalog.claim). + claim(entryId: string, label: string): boolean; + // Names of the entities this push actually publishes an entry for. An + // abstract entity, or one a binding profile pruned, is absent, so an edge + // ending on it would name an entity the catalog has no entry for. + publishedEntities: Set; + // Renders a table into the linked-resource form the catalog stores + // (knowledge_catalog.resourcePath), so the junction is addressed the same way + // an entity's backing table is. + resource(dataSource: string): string; +} + +// The entry id of one association: `.associations.`, alongside +// `.entities.` and `.metrics.`. +export function associationEntryId(modelId: string, relName: string): string { + return `${modelId}.associations.${slug(relName)}`; +} + +// The entry-id prefix a model's associations occupy, so delete reconciliation +// removes the entry of a relationship dropped from the model. +export function associationOwnedPrefix(modelId: string): string { + return `${modelId}.associations.`; +} + +/** + * One entry per many-to-many relationship, to append to the model's entries. + * + * Empty when the model declares none, so a model of only foreign-key edges is + * unchanged from before this type existed. An edge whose endpoint entity this + * push does not publish is skipped with a warning, the same way + * `relationshipLink` skips a foreign-key edge in that situation. + */ +export function associationEntries( + model: SemanticModel, modelId: string, ctx: AssociationEmitContext, + warnings: string[]): Entry[] { + const entries: Entry[] = []; + for (const rel of model.relationships ?? []) { + const assoc = rel.association; + if (!assoc) continue; + + const missing = !ctx.publishedEntities.has(rel.source.entity) ? + rel.source.entity : + !ctx.publishedEntities.has(rel.destination.entity) ? + rel.destination.entity : + undefined; + if (missing !== undefined) { + warnings.push( + `relationship '${rel.name}': endpoint entity '${missing}' is not a ` + + `published entity; the many-to-many relationship is skipped.`); + continue; + } + + const id = associationEntryId(modelId, rel.name); + if (!ctx.claim(id, `many-to-many relationship '${rel.name}'`)) continue; + entries.push({ + name: ctx.entry(id), + entryType: associationEntryTypeName(ctx), + parentEntry: ctx.anchor, + entrySource: compact({ + displayName: rel.name, + description: rel.description, + }) as Entry['entrySource'], + aspects: { + [associationAspectKey(ctx)]: { + aspectType: associationAspectTypeName(ctx), + data: associationAspectData(rel, assoc, ctx), + }, + }, + }); + } + return entries; +} + +// The aspect payload for one many-to-many relationship: its two endpoints, the +// junction table and the columns on it that reach each endpoint, the edge's own +// properties, and any AI instructions. +function associationAspectData( + rel: Relationship, assoc: Association, + ctx: AssociationEmitContext): Record { + return compact({ + fromEntity: rel.source.entity, + toEntity: rel.destination.entity, + // A model with no physical binding has no junction table to name; the + // template leaves the field optional, so omit it rather than store ''. + junction: ctx.resource(assoc.dataSource) || undefined, + keys: nonEmpty(assoc.keys), + fromColumns: nonEmpty(assoc.sourceColumns), + toColumns: nonEmpty(assoc.destinationColumns), + fields: nonEmpty( + (assoc.fields ?? + []).map(f => compact({ + name: f.name, + // An untyped field is published as Opaque, the explicit + // "type unknown" marker, so a pull recovers it as Opaque + // rather than dropping the type + // -- what the built-in schema aspect does for an entity's + // fields. + dataType: f.type ?? 'Opaque', + description: f.description, + expression: f.expression, + }))), + instructions: rel.aiContext?.instructions || undefined, + }); +} + + +// --------------------------------------------------------------------------- +// Read side: an association entry -> the IR. +// --------------------------------------------------------------------------- + +// True when an entry is one of a model's many-to-many relationships, matched by +// the entry type's id suffix so the project the type lives in need not be known +// -- which is what lets a pull keep working when the custom type is replaced by +// a built-in one. +export function isAssociationEntry(entry: Entry): boolean { + return entry.entryType?.endsWith(`/entryTypes/${ASSOCIATION_TYPE_ID}`) ?? + false; +} + +// The aspect type resource names to hydrate for an association entry. Named +// through the entry type's own project so the pull follows the type wherever it +// lives. +export function associationAspectTypes(entryTypeBase: string): string[] { + return [`${entryTypeBase}/aspectTypes/${ASSOCIATION_TYPE_ID}`]; +} + +/** + * Recovers one many-to-many relationship from its entry, the inverse of + * associationEntries. + * + * Returns undefined, with a warning, for an entry naming an endpoint that is + * not one of the model's entities, or missing a junction column list: an edge + * that cannot say what it joins is not a usable relationship, and one bad entry + * degrades itself rather than the pull. + */ +export function readAssociation( + entry: Entry, entityNames: string[], warnings: string[]): Relationship| + undefined { + const name = entry.entrySource?.displayName || idOf(entry.name); + const data = associationAspectDataOf(entry); + const known = new Set(entityNames); + + const from = str(data.fromEntity); + const to = str(data.toEntity); + for (const [side, end] of [['fromEntity', from], ['toEntity', to]] as const) { + if (!known.has(end)) { + warnings.push( + `many-to-many relationship '${name}': ${side} '${end}' is not one ` + + `of the model's entities; the relationship is skipped`); + return undefined; + } + } + + const fromColumns = stringList(data.fromColumns); + const toColumns = stringList(data.toColumns); + if (!fromColumns.length || !toColumns.length) { + const side = !fromColumns.length ? 'from' : 'to'; + warnings.push( + `many-to-many relationship '${name}': the ${ASSOCIATION_TYPE_ID} ` + + `aspect names no junction column on the ${side} end; the ` + + `relationship is skipped`); + return undefined; + } + + const association: Association = { + dataSource: dataSourceFromResource(str(data.junction)), + keys: stringList(data.keys), + sourceColumns: fromColumns, + destinationColumns: toColumns, + }; + const fields = asArray(data.fields) + .map(f => readField(f)) + .filter((f): f is Field => f !== undefined); + if (fields.length) association.fields = fields; + + // A many-to-many edge carries no columns on either ENDPOINT table: the + // columns that bind it are the junction's, above. Empty lists here are what + // the loader produces for an authored `association`, so the two agree. + const relationship: Relationship = { + name, + source: {entity: from, columns: []}, + destination: {entity: to, columns: []}, + association, + }; + const description = entry.entrySource?.description; + if (description !== undefined && description !== '') { + relationship.description = description; + } + const instructions = str(data.instructions); + if (instructions) relationship.aiContext = {instructions} as AiContext; + return relationship; +} + +// One edge property from its aspect record. A record with no name is dropped: a +// nameless field cannot be referenced and would not survive a reload. +function readField(f: any): Field|undefined { + const name = str(f?.name); + if (!name) return undefined; + const field: Field = {name}; + const expression = str(f?.expression); + if (expression) field.expression = expression; + const dataType = str(f?.dataType); + if ((DATA_TYPES as readonly string[]).includes(dataType)) { + field.type = dataType as DataType; + } + const description = str(f?.description); + if (description) field.description = description; + return field; +} + + +// --------------------------------------------------------------------------- +// Local helpers (see the file header on why they are not shared). +// --------------------------------------------------------------------------- + +// The association aspect's `data` from an entry, matched by the aspect key's +// `.semantic-association` suffix or the aspectType's +// `/aspectTypes/semantic-association` suffix, so it is found whichever project +// the type was provisioned in. +function associationAspectDataOf(entry: Entry): Record { + for (const [key, aspect] of Object.entries(entry.aspects ?? {})) { + if (key.endsWith(`.${ASSOCIATION_TYPE_ID}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${ASSOCIATION_TYPE_ID}`)) { + return aspect.data ?? {}; + } + } + return {}; +} + +// The dotted `project.dataset.table` form of a stored linked resource, the +// inverse of knowledge_catalog.resourcePath. Anything that is not a BigQuery +// table URI comes back as it was stored. +function dataSourceFromResource(resource: string): string { + const value = resource.trim(); + const m = value.match( + /^\/\/bigquery\.googleapis\.com\/projects\/([^/]+)\/datasets\/([^/]+)\/tables\/([^/]+)$/); + return m ? `${m[1]}.${m[2]}.${m[3]}` : value; +} + +// Entry ids allow letters, numbers, underscores, hyphens, and periods. +function slug(s: string): string { + return s.replace(/[^A-Za-z0-9_.-]/g, '_'); +} + +// The id segment of a full entry resource name. +function idOf(name: string): string { + return name.split('/').pop() ?? name; +} + +// Drops undefined-valued keys so the emitted aspect (and its golden) only shows +// fields the model actually set. +function compact>(obj: T): T { + const out: Record = {}; + for (const [k, v] of Object.entries(obj)) { + if (v !== undefined) out[k] = v; + } + return out as T; +} + +function asArray(value: any): any[] { + return Array.isArray(value) ? value : []; +} + +// The non-empty strings of an array value, dropping non-string and empty +// members so a degenerate '' does not round-trip. +function stringList(value: any): string[] { + return asArray(value).filter( + (s): s is string => typeof s === 'string' && s !== ''); +} + +function str(value: any): string { + return typeof value === 'string' ? value : ''; +} + +function nonEmpty(list: T[]): T[]|undefined { + return list.length ? list : undefined; +} diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index 31e65d45..c32e9d56 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -33,8 +33,10 @@ // (re-derived from its expression, as the loader does, when the catalog holds // one, else the value the emitter persisted), the model's deployment targets // (from the semantic-model aspect, back into the GOOGLE `custom_extensions` -// block), and 1:1 / 1:N relationships (from the `schema-join` entry links a -// pull fetched -- see `modelsFromCatalogResources`'s `entryLinks` argument). +// block), 1:1 / 1:N relationships (from the `schema-join` entry links a pull +// fetched -- see `modelsFromCatalogResources`'s `entryLinks` argument), and +// many-to-many relationships (from the `semantic-association` entries, which +// carry the junction table and both column pairs -- see kc_associations.ts). // The per-field `semantics` block -- field/metric expressions and the DIMENSION // role // -- is gated off the catalog by default: the emitter writes it only under @@ -43,19 +45,21 @@ // push -> pull drops them. It cannot recover what the emitter never writes: // `ai_context.synonyms`/`examples` and field-level `ai_context` (only // model/entity/metric `instructions` are persisted, via `guidelines`), -// `importedExpression`/`importedDialect` (the vendor-dialect SQL), and -// many-to-many (association) relationships (whose edge lives only in the -// BigQuery property graph). A `String`- or `Opaque`-typed METRIC also reads +// `importedExpression`/`importedDialect` (the vendor-dialect SQL). A `String`- +// or `Opaque`-typed METRIC also reads // back un-typed: the metric aspect persists only `dataType` (both collapse to // `STRING`, with no metadataType to disambiguate), so -- as with a plain STRING -// field -- the reader leaves it un-typed rather than guess. Relationship NAMES -// come back normalized (lowercased/hyphenated), since the emitter encodes the -// name only in the link id (via `linkSlug`), not in the join aspect. +// field -- the reader leaves it un-typed rather than guess. A FOREIGN-KEY +// relationship's NAME comes back normalized (lowercased/hyphenated), since the +// emitter encodes it only in the link id (via `linkSlug`), not in the join +// aspect; a many-to-many one keeps its authored name, which its entry carries +// verbatim as a display name. import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {Action, AiContext, CustomExtension, DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; import {isActionEntry, readAction} from './kc_actions'; +import {isAssociationEntry, readAssociation} from './kc_associations'; import {referencedEntityNames} from './sql_expr_utils'; export interface ReadResult { @@ -86,8 +90,10 @@ export function modelsFromCatalogResources( const metricEntries = entries.filter(e => semanticType(e) === 'semantic-metric'); // Actions have no built-in system type; `kc_actions.ts` owns the custom one - // and recognizes an entry carrying it. + // and recognizes an entry carrying it. A many-to-many relationship is the + // same arrangement, in `kc_associations.ts`. const actionEntries = entries.filter(isActionEntry); + const associationEntries = entries.filter(isAssociationEntry); if (!anchors.length) { warnings.push('no semantic-model entry found; nothing to reconstruct'); @@ -122,11 +128,16 @@ export function modelsFromCatalogResources( entityEntriesForModel.forEach( (e, i) => entityByEntryId.set(idOf(e.name), entities[i])); - // Relationships come from the schema-join entry links whose two endpoints - // are both this model's entity entries (M:N edges were never published -- - // they live only in the BigQuery property graph -- so stay absent here). - const relationships = - readRelationships(entryLinks, name, entityByEntryId, warnings); + // A direct foreign-key relationship comes from the schema-join entry link + // whose two endpoints are both this model's entity entries. A many-to-many + // one is not a link but an entry of its own, so it is read separately and + // appended; the two together are the model's edges. + const relationships = [ + ...readRelationships(entryLinks, name, entityByEntryId, warnings), + ...childrenOf(anchor.name, associationEntries) + .map(e => readAssociation(e, entityNames, warnings)) + .filter((r): r is Relationship => r !== undefined), + ]; const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; @@ -147,8 +158,8 @@ export function modelsFromCatalogResources( // Flag children that resolved to no anchor at all (only possible with // multiple anchors, where the sole-anchor fallback does not apply). if (!soleAnchor) { - for (const child of [...entityEntries, ...metricEntries, - ...actionEntries]) { + for (const child of [...entityEntries, ...metricEntries, ...actionEntries, + ...associationEntries]) { if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { warnings.push(`entry '${ child.name}' has no resolvable parent semantic-model; omitted`); diff --git a/toolbox/mdcode/src/libts/semantic/kc_custom_types.ts b/toolbox/mdcode/src/libts/semantic/kc_custom_types.ts index e630cc2a..decbcf83 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_custom_types.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_custom_types.ts @@ -21,7 +21,8 @@ // TO ADD A NEW CUSTOM TYPE. Append a record. Provisioning, naming and the init // wiring are generic over this list, so no other file in this directory needs // to change; what does need writing is the encoding that fills the aspect, the -// way kc_actions.ts does for `semantic-action`. +// way kc_actions.ts does for `semantic-action` and kc_associations.ts does for +// `semantic-association`. // // A CUSTOM TYPE IS ONE ENTRY TYPE PLUS ONE ASPECT TYPE that share an id, the // way the built-in `semantic-metric` entry type and aspect type share theirs. @@ -196,6 +197,173 @@ const ACTION_ASPECT_TYPE: Omit = { }, }; +// The id of the association type. An association is the junction table backing +// a many-to-many relationship; kc_associations.ts holds the encoding that fills +// its aspect. +export const ASSOCIATION_TYPE_ID = 'semantic-association'; + +// The aspect that carries a many-to-many relationship. +// +// The built-in `schema-join` entry link already models a relationship, but only +// a direct foreign key: it holds ONE source/target column pair. A many-to-many +// edge is two joins through a third table, so it does not fit, and a custom +// entry LINK type is not available -- Dataplex accepts only its own link types. +// That leaves an entry, which is what this type is: one per many-to-many +// relationship, parented to the model anchor beside the entities and metrics. +// +// The endpoints are recorded as entity NAMES rather than as entry references +// because the two entity entries are already the link endpoints a consumer +// resolves by name everywhere else in this encoding (a metric's `entity`, an +// action parameter's `type`), and a name survives the project-number +// normalization that rewrites resource names on the way back. +// +// `fields` is the association's own properties -- an enrollment's grade, which +// belongs to neither endpoint. They ride here rather than in the built-in +// `schema` aspect for the same reason `instructions` does not use `guidelines`: +// a pull derives which aspect types to hydrate from the project the ENTRY type +// lives in, and a custom entry type lives in the destination project, where no +// built-in aspect type exists. +const ASSOCIATION_ASPECT_TYPE: Omit = { + displayName: 'Semantic Association', + description: + 'A many-to-many relationship in a semantic model: the two entities it ' + + 'pairs, and the junction table that holds the pairs.', + metadataTemplate: { + name: ASSOCIATION_TYPE_ID, + type: 'record', + recordFields: [ + { + index: 1, + name: 'fromEntity', + type: 'string', + constraints: {required: true}, + annotations: { + displayName: 'From Entity', + description: + 'Name of the entity at the source end. It is the SOURCE of the ' + + 'edge in a property graph.', + }, + }, + { + index: 2, + name: 'toEntity', + type: 'string', + constraints: {required: true}, + annotations: { + displayName: 'To Entity', + description: + 'Name of the entity at the destination end. It is the ' + + 'DESTINATION of the edge in a property graph.', + }, + }, + { + index: 3, + name: 'junction', + type: 'string', + annotations: { + displayName: 'Junction Table', + description: + 'Resource name of the table holding the pairs, one row per ' + + '(from, to). Empty on a model with no physical binding.', + }, + }, + { + index: 4, + name: 'keys', + type: 'array', + arrayItems: {name: 'key', type: 'string'}, + annotations: { + displayName: 'Keys', + description: 'The edge\'s own key columns on the junction table.', + }, + }, + { + index: 5, + name: 'fromColumns', + type: 'array', + arrayItems: {name: 'column', type: 'string'}, + annotations: { + displayName: 'From Columns', + description: + 'Junction-table columns referencing the from entity\'s key.', + }, + }, + { + index: 6, + name: 'toColumns', + type: 'array', + arrayItems: {name: 'column', type: 'string'}, + annotations: { + displayName: 'To Columns', + description: + 'Junction-table columns referencing the to entity\'s key.', + }, + }, + { + index: 7, + name: 'fields', + type: 'array', + arrayItems: { + name: 'field', + type: 'record', + recordFields: [ + { + index: 1, + name: 'name', + type: 'string', + constraints: {required: true}, + annotations: {displayName: 'Name'}, + }, + { + index: 2, + name: 'dataType', + type: 'string', + annotations: { + displayName: 'Data Type', + description: 'The field\'s logical datatype.', + }, + }, + { + index: 3, + name: 'description', + type: 'string', + annotations: {displayName: 'Description'}, + }, + { + index: 4, + name: 'expression', + type: 'string', + annotations: { + displayName: 'Expression', + description: + 'The junction-table column the field binds to. Empty on ' + + 'a model with no physical binding.', + }, + }, + ], + }, + annotations: { + displayName: 'Fields', + description: + 'Properties of the pairing itself, belonging to neither ' + + 'endpoint (an enrollment\'s grade).', + }, + }, + { + index: 8, + name: 'instructions', + type: 'string', + annotations: { + displayName: 'Instructions', + description: + 'Guidance for AI consumers (the relationship\'s ai_context ' + + 'instructions).', + }, + }, + ], + }, +}; + // Every type kcmd provisions. This list is the whole of what is custom. export const CUSTOM_TYPES: readonly CustomType[] = [ { @@ -206,6 +374,16 @@ export const CUSTOM_TYPES: readonly CustomType[] = [ }, aspectType: ACTION_ASPECT_TYPE, }, + { + id: ASSOCIATION_TYPE_ID, + entryType: { + displayName: 'Semantic Association', + description: + 'A many-to-many relationship in a semantic model, backed by a ' + + 'junction table.', + }, + aspectType: ASSOCIATION_ASPECT_TYPE, + }, ]; // Custom types are provisioned at `global` so an entry group in any region can diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index 2d4b97ef..0b7ac1ae 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -42,15 +42,20 @@ // system-type templates yet, so they are gated behind // KcGenerateOptions.emitExpressions (off by default) and omitted above. // -// Relationships become `schema-join` entry links between the two entity entries. -// schema-join is a built-in, undirected entry link type in `dataplex-types/global` -// whose required `schema-join` aspect carries the join detail (the paired join -// columns, JOIN vs FOREIGN_KEY, USER inference). The join's direction -- which -// side holds the foreign key -- is preserved inside that aspect, not by the link. -// Many-to-many (association / junction-table) edges are not emitted yet: a -// junction is two joins through a third table, which schema-join's single -// source/target pair does not model; the emitter warns and skips them (the edge -// still lives in the BigQuery property graph, see bigquery.ts). +// A direct foreign-key relationship becomes a `schema-join` entry link between +// the two entity entries. schema-join is a built-in, undirected entry link type +// in `dataplex-types/global` whose required `schema-join` aspect carries the +// join detail (the paired join columns, JOIN vs FOREIGN_KEY, USER inference). +// The join's direction -- which side holds the foreign key -- is preserved +// inside that aspect, not by the link. +// +// A MANY-TO-MANY relationship is not a link at all. A junction is two joins +// through a third table, which schema-join's single source/target pair does not +// model, and Dataplex has no custom entry LINK types to define one with. It +// publishes as an ENTRY instead, one per relationship parented to the anchor, +// using the same custom-type mechanism as an action: `kc_custom_types.ts` +// declares the `semantic-association` pair and `kc_associations.ts` encodes the +// aspect. This module only appends the entries it returns. // import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; @@ -58,6 +63,7 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {googleDeploymentTargets} from './deployment_target'; import {AiContext, DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; import {actionEntries, actionOwnedPrefix} from './kc_actions'; +import {associationEntries, associationOwnedPrefix} from './kc_associations'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -101,11 +107,12 @@ export interface KcResources { /** * Generates the Knowledge Catalog resources for a semantic model. * - * Returns the entries (model anchor first, then entities and metrics), the - * schema-join entry links for the model's relationships, and any warnings - * collected while mapping the IR (missing keys, un-typed metrics, skipped M:N - * relationships). The resources reference the built-in system types; they do not - * create them. + * Returns the entries (model anchor first, then entities, metrics, actions and + * many-to-many relationships), the schema-join entry links for the model's + * foreign-key relationships, and any warnings collected while mapping the IR + * (missing keys, un-typed metrics, edges whose endpoint entity is unpublished). + * The resources reference the built-in system types, and the custom ones `kcmd + * init` provisions; they do not create either. */ export function generateCatalogResources( model: SemanticModel, opts: KcGenerateOptions): KcResources { @@ -216,7 +223,24 @@ export function generateCatalogResources( publishedEntities: new Set(entityEntryName.keys()), }, warnings)); - // Relationships map to schema-join entry links between their endpoint entries. + // One entry per many-to-many relationship, for the same reason: a junction + // has no built-in type either, so `kc_associations.ts` fills the custom + // `semantic-association` aspect declared in `kc_custom_types.ts`. It returns + // nothing when every relationship is a direct foreign key. + entries.push(...associationEntries(model, modelId, { + project: opts.project, + entry: (id: string) => names.entry(id), + anchor: modelEntryName, + claim: (id: string, label: string) => + claim(seen, id, 'entry', label, warnings), + publishedEntities: new Set(entityEntryName.keys()), + // The junction table is addressed the same way an entity's backing table + // is, so the emitter's own mapping is what renders it. + resource: resourcePath, + }, warnings)); + + // A direct foreign-key relationship maps to a schema-join entry link between + // its endpoint entries. A many-to-many one became an entry above. const entryLinks: EntryLink[] = []; const seenLinks = new Set(); for (const rel of relationships) { @@ -230,35 +254,30 @@ export function generateCatalogResources( entryLinks, warnings: [...new Set(warnings)], // Ossie ids are dotted: `.entities.` / `.metrics.` - // / `.actions.`. + // / `.actions.` / `.associations.`. ownedPrefixes: [ `${modelId}.entities.`, `${modelId}.metrics.`, actionOwnedPrefix(modelId), + associationOwnedPrefix(modelId), ], }; } -// Builds the schema-join entry link for one relationship, or undefined when it -// cannot be published. Skipped, each with a warning: a many-to-many -// (association) edge -- a junction is two joins, which the single source/target -// schema-join does not model; an edge whose endpoint entity was not emitted -// (e.g. skipped for a duplicate id); and a column-less (purely logical) edge, -// whose join columns must be added to the model before it can publish. The -// BigQuery property graph still carries the association and (once bound) direct -// edges. +// Builds the schema-join entry link for one direct foreign-key relationship, or +// undefined when it cannot be published. A many-to-many edge returns undefined +// without a warning: it is not a link, and `associationEntries` has already +// published it as an entry. Two cases are skipped WITH a warning: an edge whose +// endpoint entity was not emitted (e.g. skipped for a duplicate id), and a +// column-less (purely logical) edge, whose join columns must be added to the +// model before it can publish. The BigQuery property graph still carries the +// latter once bound. function relationshipLink( names: Namer, model: SemanticModel, rel: Relationship, entityEntryName: Map, seenLinks: Set, warnings: string[]): EntryLink|undefined { - if (rel.association) { - warnings.push( - `relationship '${rel.name}': many-to-many (association) edges are not ` + - `published to Knowledge Catalog yet; the edge lives in the BigQuery ` + - `property graph.`); - return undefined; - } + if (rel.association) return undefined; const src = entityEntryName.get(rel.source.entity); const dst = entityEntryName.get(rel.destination.entity); if (!src || !dst) { diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index 5ea34b26..26ed9500 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -20,7 +20,8 @@ import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; import {SemanticModel} from './ir'; import {actionAspectTypes} from './kc_actions'; -import {ACTION_TYPE_ID} from './kc_custom_types'; +import {associationAspectTypes} from './kc_associations'; +import {ACTION_TYPE_ID, ASSOCIATION_TYPE_ID} from './kc_custom_types'; import {idOf, linkDedupKey, modelsFromCatalogResources} from './kc_converter'; export interface KcPullOptions { @@ -165,6 +166,10 @@ function semanticAspectTypes(entryType: string): string[]|undefined { // `typeBase` already points there, and kc_actions.ts names the aspects // to fetch beneath it. return actionAspectTypes(typeBase); + case ASSOCIATION_TYPE_ID: + // Likewise for a many-to-many relationship: its entry type is custom, and + // kc_associations.ts names the one aspect that holds the junction. + return associationAspectTypes(typeBase); default: return undefined; } diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.knowledge_catalog.golden.json new file mode 100644 index 00000000..a7f59c87 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.knowledge_catalog.golden.json @@ -0,0 +1,139 @@ +{ + "entries": [ + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-model", + "entrySource": { + "displayName": "school_graph", + "description": "Students, courses, and the enrollments that pair them" + }, + "aspects": { + "dataplex-types.global.semantic-model": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-model", + "data": {} + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph.entities.students", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph", + "entrySource": { + "displayName": "students" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/bei_semantic_ir_verify/tables/students" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "student_id", + "dataType": "STRING", + "metadataType": "OTHER" + }, + { + "name": "name", + "dataType": "STRING", + "metadataType": "OTHER" + } + ], + "primaryKey": { + "fields": [ + "student_id" + ] + } + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph.entities.courses", + "entryType": "projects/dataplex-types/locations/global/entryTypes/semantic-entity", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph", + "entrySource": { + "displayName": "courses" + }, + "aspects": { + "dataplex-types.global.semantic-entity": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/semantic-entity", + "data": { + "source": { + "resources": [ + "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/bei_semantic_ir_verify/tables/courses" + ] + } + } + }, + "dataplex-types.global.schema": { + "aspectType": "projects/dataplex-types/locations/global/aspectTypes/schema", + "data": { + "fields": [ + { + "name": "course_id", + "dataType": "STRING", + "metadataType": "OTHER" + }, + { + "name": "title", + "dataType": "STRING", + "metadataType": "OTHER" + } + ], + "primaryKey": { + "fields": [ + "course_id" + ] + } + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph.associations.enrollment", + "entryType": "projects/sqlgen-testing/locations/global/entryTypes/semantic-association", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph", + "entrySource": { + "displayName": "enrollment" + }, + "aspects": { + "sqlgen-testing.global.semantic-association": { + "aspectType": "projects/sqlgen-testing/locations/global/aspectTypes/semantic-association", + "data": { + "fromEntity": "students", + "toEntity": "courses", + "junction": "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/bei_semantic_ir_verify/tables/enrollment", + "keys": [ + "enrollment_id" + ], + "fromColumns": [ + "student_id" + ], + "toColumns": [ + "course_id" + ], + "fields": [ + { + "name": "grade", + "dataType": "Opaque", + "description": "Letter grade", + "expression": "enrollment.grade" + } + ] + } + } + } + } + ], + "entryLinks": [], + "warnings": [] +} diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.osi.golden.yaml new file mode 100644 index 00000000..95e49a68 --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.osi.golden.yaml @@ -0,0 +1,54 @@ +version: 0.2.0.dev0/google +semantic_model: + - name: school_graph + description: Students, courses, and the enrollments that pair them + entities: + - name: students + source: sqlgen-testing.bei_semantic_ir_verify.students + primary_key: + - student_id + fields: + - name: student_id + expression: + dialects: + - dialect: BIGQUERY + expression: students.student_id + - name: name + expression: + dialects: + - dialect: BIGQUERY + expression: students.name + - name: courses + source: sqlgen-testing.bei_semantic_ir_verify.courses + primary_key: + - course_id + fields: + - name: course_id + expression: + dialects: + - dialect: BIGQUERY + expression: courses.course_id + - name: title + expression: + dialects: + - dialect: BIGQUERY + expression: courses.title + relationships: + - name: enrollment + from: students + to: courses + association: + source: sqlgen-testing.bei_semantic_ir_verify.enrollment + keys: + - enrollment_id + from_columns: + - student_id + to_columns: + - course_id + fields: + - name: grade + expression: + dialects: + - dialect: BIGQUERY + expression: enrollment.grade + description: Letter grade diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.pull.golden.yaml new file mode 100644 index 00000000..d232ccff --- /dev/null +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.pull.golden.yaml @@ -0,0 +1,44 @@ +# (no warnings) +version: 0.2.0.dev0/google +semantic_model: + - name: school_graph + description: Students, courses, and the enrollments that pair them + entities: + - name: students + source: sqlgen-testing.bei_semantic_ir_verify.students + primary_key: + - student_id + fields: + - name: student_id + datatype: Opaque + - name: name + datatype: Opaque + - name: courses + source: sqlgen-testing.bei_semantic_ir_verify.courses + primary_key: + - course_id + fields: + - name: course_id + datatype: Opaque + - name: title + datatype: Opaque + relationships: + - name: enrollment + from: students + to: courses + association: + source: sqlgen-testing.bei_semantic_ir_verify.enrollment + keys: + - enrollment_id + from_columns: + - student_id + to_columns: + - course_id + fields: + - name: grade + expression: + dialects: + - dialect: BIGQUERY + expression: enrollment.grade + datatype: Opaque + description: Letter grade diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index 0a4e379e..b9bd050d 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -282,25 +282,114 @@ describe('relationship recovery (schema-join links -> IR)', () => { .toBe('places-order'); }); - test('a many-to-many (association) relationship is not recovered', () => { + test('a many-to-many relationship round-trips through its own entry', () => { + // It is not a schema-join link but a semantic-association entry, so it + // comes back from `entries` rather than `entryLinks` -- and keeps its + // authored name, which the entry carries verbatim. const model: SemanticModel = { name: 'sales', entities: twoEntities, relationships: [{ - name: 'enrolls', - source: {entity: 'orders', columns: ['o_custkey']}, - destination: {entity: 'customer', columns: ['c_custkey']}, + name: 'Promoted_By', + source: {entity: 'orders', columns: []}, + destination: {entity: 'customer', columns: []}, + description: 'which customers an order reached', association: { dataSource: 'p.d.junction', keys: ['id'], sourceColumns: ['j_orderkey'], destinationColumns: ['j_custkey'], + fields: [{name: 'discount', expression: 'j.discount', + type: 'Decimal'}], }, }], metrics: [], }; - // The emitter never publishes M:N, so no schema-join link exists to read. - expect(roundTrip(model).models[0].relationships).toEqual([]); + expect(roundTrip(model).models[0].relationships).toEqual([ + model.relationships[0] + ]); + }); + + test('a many-to-many relationship and a foreign key coexist', () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [ + { + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }, + { + name: 'promoted-by', + source: {entity: 'orders', columns: []}, + destination: {entity: 'customer', columns: []}, + association: { + dataSource: 'p.d.junction', + keys: [], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + }, + }, + ], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships.map(r => r.name)) + .toEqual(['places', 'promoted-by']); + }); + + test('an association entry naming an unknown entity is skipped', () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'promoted-by', + source: {entity: 'orders', columns: []}, + destination: {entity: 'customer', columns: []}, + association: { + dataSource: 'p.d.junction', + keys: [], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + }, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const assoc = entries.find( + e => e.entryType.endsWith('/entryTypes/semantic-association'))!; + assoc.aspects!['dest.global.semantic-association'].data!.toEntity = + 'ghost'; + const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); + expect(models[0].relationships).toEqual([]); + expect(warnings.some(w => w.includes('ghost'))).toBe(true); + }); + + test('an association entry with no junction columns is skipped', () => { + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'promoted-by', + source: {entity: 'orders', columns: []}, + destination: {entity: 'customer', columns: []}, + association: { + dataSource: 'p.d.junction', + keys: [], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + }, + }], + metrics: [], + }; + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + const assoc = entries.find( + e => e.entryType.endsWith('/entryTypes/semantic-association'))!; + delete assoc.aspects!['dest.global.semantic-association'] + .data!.toColumns; + const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); + expect(models[0].relationships).toEqual([]); + expect(warnings.some(w => w.includes('no junction column'))).toBe(true); }); test( @@ -741,9 +830,9 @@ describe('metric expression referencing no known entity', () => { // entries and entry links a push produced) back through the reader and // serializes the reconstructed IR to `.pull.golden.yaml`. Open that // next to the fixture's `.osi.golden.yaml` to see, as whole files, what a -// Knowledge Catalog round trip preserves (including 1:1 / 1:N relationships and -// deployment targets) and what it drops (keys, ai_context, labels, vendor SQL, -// M:N relationships). +// Knowledge Catalog round trip preserves (including relationships of both +// arities and deployment targets) and what it drops (ai_context beyond +// instructions, labels, vendor SQL). // // Regenerate after an intentional reader/serializer change: // UPDATE_GOLDENS=1 npx bun test ./tests/libts/semantic/kc_converter.test.ts @@ -753,6 +842,7 @@ describe( 'sales_bq_graph_target.yaml', 'star_orders_customer.yaml', 'tpcds_date_edge.yaml', + 'school_manytomany.yaml', ]; const kcGoldenPath = (fixture: string) => path.join( FIXTURES, @@ -797,14 +887,15 @@ describe( // the documented losses and normalizations applied (`stripToKcFloor`). Applying // that same reduction to BOTH sides makes `toEqual` flag only UNDOCUMENTED // divergence, so a new emitter/reader regression (a dropped column, a lost -// description, an un-stripped M:N edge) fails here even though every individual -// loss is already pinned by a targeted test above. +// description, a junction that stopped round-tripping) fails here even though +// every individual loss is already pinned by a targeted test above. describe( 'symmetry: an emit -> read round trip drops only documented fields', () => { const CORPUS = [ 'sales_bq_graph_target.yaml', 'star_orders_customer.yaml', 'tpcds_date_edge.yaml', + 'school_manytomany.yaml', ]; // Same load defaults as the OSI / KC / pull goldens. const LOAD = {defaultProject: 'sqlgen-testing', defaultDataset: 'demo'}; @@ -893,13 +984,42 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { } } - // M:N (association) edges are never published; a 1:1 / 1:N name comes back - // normalized via the emitter's slug (its exact form is pinned above). - m.relationships = m.relationships.filter(r => !r.association).map(r => { + // The two arities lose different things, because they are published as + // different resources. A direct foreign key is a schema-join entry LINK, + // which carries no name field and no aspect for guidelines, so the name comes + // back normalized via the emitter's slug and ai_context is gone. A + // many-to-many edge is an ENTRY of the custom semantic-association type, + // which carries its own display name and folds instructions and the edge's + // fields into its own aspect, so all three survive. + m.relationships = m.relationships.map(r => { const rel = structuredClone(r); - rel.name = linkNamePrefix(rel.name); - delete rel.aiContext; delete rel.customExtensions; + if (!rel.association) { + rel.name = linkNamePrefix(rel.name); + delete rel.aiContext; + return rel; + } + floorAiContext(rel); + // An empty field list is written as nothing and reads back absent. + if (rel.association.fields && !rel.association.fields.length) { + delete rel.association.fields; + } + for (const f of rel.association.fields ?? []) { + delete f.aiContext; + delete f.importedExpression; + delete f.importedDialect; + delete f.customExtensions; + // The association aspect has no slot for a display label or a dimension + // role. It does store the field's expression -- unlike an entity field, + // whose expression is gated off because the BUILT-IN schema template has + // nowhere to put it; this template is ours, so it carries one. + delete f.label; + delete f.dimension; + // An un-typed field is published as Opaque, the explicit "type unknown" + // marker, and reads back as Opaque. Unlike an entity field, `String` is + // stored verbatim rather than collapsing into a bare STRING. + if (f.type === undefined) f.type = 'Opaque'; + } return rel; }); diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts index 5452ec55..6e23b46c 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts @@ -31,11 +31,13 @@ const FIXTURES = path.join(__dirname, 'fixtures'); // sales_bq_graph_target -> model aspect deploymentTargets + un-typed metric // (dataType fallback); star_orders_customer -> a direct-FK relationship // (schema-join link) + multiple entities/metrics; tpcds_date_edge -> -// temporal field types. +// temporal field types; school_manytomany -> a junction-backed edge, which +// is a semantic-association entry rather than a link. const CORPUS = [ 'sales_bq_graph_target.yaml', 'star_orders_customer.yaml', 'tpcds_date_edge.yaml', + 'school_manytomany.yaml', ]; // A fixed destination + default (dataplex-types/global) system types, so the diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts index d94992c1..dcdc1ecc 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -557,23 +557,14 @@ describe('relationships map to schema-join entry links', () => { expect(warnings.length).toBe(0); }); - test( - 'a many-to-many (association) edge is skipped and warned, no link', - () => { - const model = directFkModel(); - model.relationships[0].association = { - dataSource: 'p.d.order_customer', - keys: ['id'], - sourceColumns: ['o_key'], - destinationColumns: ['c_key'], - }; - const {entryLinks, warnings} = generateCatalogResources(model, OPTS); - expect(entryLinks.length).toBe(0); - expect(warnings.some( - w => w.includes('orders-to-customer') && - w.includes('many-to-many'))) - .toBe(true); - }); + test('a many-to-many edge produces no link (it is an entry instead)', () => { + const {entryLinks, warnings} = generateCatalogResources(mnModel(), OPTS); + expect(entryLinks.length).toBe(0); + // Silently, because the edge is published -- as an entry, asserted in the + // many-to-many describe below. A warning here would say a supported + // construct was dropped. + expect(warnings.length).toBe(0); + }); test('an edge to an unpublished entity is skipped and warned', () => { const model = directFkModel(); @@ -622,6 +613,169 @@ describe('relationships map to schema-join entry links', () => { }); +// The same two entities paired through a junction table instead of a foreign +// key: an order is on many promotions and a promotion covers many orders, so +// the pairs live in `p.d.order_promotion` with a `discount` of their own. +function mnModel(): SemanticModel { + return { + name: 'm', + metrics: [], + entities: [ + {name: 'orders', dataSource: 'p.d.orders', keys: ['o_key'], fields: []}, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: ['c_key'], + fields: [] + }, + ], + relationships: [{ + name: 'order-customer', + source: {entity: 'orders', columns: []}, + destination: {entity: 'customer', columns: []}, + description: 'which customers an order reached', + association: { + dataSource: 'p.d.order_customer', + keys: ['id'], + sourceColumns: ['j_orderkey'], + destinationColumns: ['j_custkey'], + fields: [{name: 'discount', expression: 'j.discount', type: 'Decimal'}], + }, + }], + }; +} + +// The sole semantic-association entry of a generated model. +function associationEntry(model: SemanticModel) { + const {entries} = generateCatalogResources(model, OPTS); + const found = + entries.filter(e => e.entryType.endsWith('/entryTypes/semantic-association')); + expect(found.length).toBe(1); + return found[0]; +} + +// A many-to-many relationship has no built-in Knowledge Catalog type: it is not +// a schema-join (a junction is two joins) and Dataplex has no custom entry LINK +// types. It publishes as an entry of the custom `semantic-association` type +// instead, the same mechanism actions use. See kc_associations.ts. +describe('a many-to-many relationship becomes a semantic-association entry', () => { + test('the entry is typed, named and parented to the model anchor', () => { + const entry = associationEntry(mnModel()); + expect(entry.name!.endsWith('/entries/m.associations.order-customer')) + .toBe(true); + // The custom type lives in the DESTINATION project (kcmd init creates it + // there), not under dataplex-types with the built-in types. + expect(entry.entryType) + .toBe( + 'projects/dest-proj/locations/global/entryTypes/semantic-association'); + expect(entry.parentEntry!.endsWith('/entries/m')).toBe(true); + // The authored name rides the entry source verbatim, so unlike a + // schema-join relationship it is not normalized on the way back. + expect(entry.entrySource!.displayName).toBe('order-customer'); + expect(entry.entrySource!.description) + .toBe('which customers an order reached'); + }); + + test('the aspect carries both endpoints, the junction and its columns', () => { + const entry = associationEntry(mnModel()); + const aspect = entry.aspects!['dest-proj.global.semantic-association']; + expect(aspect.aspectType) + .toBe( + 'projects/dest-proj/locations/global/aspectTypes/semantic-association'); + const data = aspect.data!; + expect(data.fromEntity).toBe('orders'); + expect(data.toEntity).toBe('customer'); + // The junction is addressed the way an entity's backing table is: the + // BigQuery linked-resource URI, not the dotted form. + expect(data.junction) + .toBe( + '//bigquery.googleapis.com/projects/p/datasets/d/tables/order_customer'); + expect(data.keys).toEqual(['id']); + expect(data.fromColumns).toEqual(['j_orderkey']); + expect(data.toColumns).toEqual(['j_custkey']); + }); + + test('edge properties ride the association aspect, not a schema aspect', () => { + // The entry's type is custom, so the built-in `schema` aspect type is not + // available to it (a pull derives the aspect base from the entry type's + // project). The edge's own fields therefore live on this aspect. + const entry = associationEntry(mnModel()); + expect(Object.keys(entry.aspects!)).toEqual([ + 'dest-proj.global.semantic-association' + ]); + const data = entry.aspects!['dest-proj.global.semantic-association'].data!; + expect(data.fields).toEqual([ + {name: 'discount', dataType: 'Decimal', expression: 'j.discount'}, + ]); + }); + + test('an untyped edge property is published as Opaque', () => { + const model = mnModel(); + delete model.relationships[0].association!.fields![0].type; + const data = associationEntry(model) + .aspects!['dest-proj.global.semantic-association'] + .data!; + expect(data.fields[0].dataType).toBe('Opaque'); + }); + + test('ai_context.instructions ride the association aspect too', () => { + const model = mnModel(); + model.relationships[0].aiContext = {instructions: 'one row per pairing'}; + const data = associationEntry(model) + .aspects!['dest-proj.global.semantic-association'] + .data!; + expect(data.instructions).toBe('one row per pairing'); + // Not the built-in guidelines aspect, which this entry type cannot require. + expect(Object.keys(associationEntry(model).aspects!)).toEqual([ + 'dest-proj.global.semantic-association' + ]); + }); + + test('the associations prefix is owned, so a dropped edge is reconciled', () => { + const {ownedPrefixes} = generateCatalogResources(mnModel(), OPTS); + expect(ownedPrefixes).toContain('m.associations.'); + }); + + test('a model of only foreign-key edges emits no association entry', () => { + const model = mnModel(); + delete model.relationships[0].association; + model.relationships[0].source.columns = ['custkey']; + model.relationships[0].destination.columns = ['c_key']; + const {entries, ownedPrefixes} = generateCatalogResources(model, OPTS); + expect(entries.some( + e => e.entryType.endsWith('/entryTypes/semantic-association'))) + .toBe(false); + // The prefix is still owned, so an edge deleted from the model has its + // entry removed on the next push. + expect(ownedPrefixes).toContain('m.associations.'); + }); + + test('an edge to an unpublished entity is skipped and warned', () => { + const model = mnModel(); + model.relationships[0].destination.entity = 'ghost'; + const {entries, warnings} = generateCatalogResources(model, OPTS); + expect(entries.some( + e => e.entryType.endsWith('/entryTypes/semantic-association'))) + .toBe(false); + expect(warnings.some( + w => w.includes('order-customer') && w.includes('ghost'))) + .toBe(true); + }); + + test('a logical-only junction omits the table rather than storing a blank', + () => { + const model = mnModel(); + model.relationships[0].association!.dataSource = ''; + const data = associationEntry(model) + .aspects!['dest-proj.global.semantic-association'] + .data!; + expect('junction' in data).toBe(false); + // The logical shape survives: the edge still says what it pairs. + expect(data.fromColumns).toEqual(['j_orderkey']); + }); +}); + + describe('a purely logical model (no physical binding) emits cleanly', () => { // A Knowledge-Catalog-only model governs meaning with no binding: every // entity has an empty dataSource and its fields carry no expression. The diff --git a/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts index c0b1674e..474daa9d 100644 --- a/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts @@ -355,6 +355,7 @@ describe('golden OSI document: each corpus fixture serializes to its exact YAML' 'sales_bq_graph_target.yaml', 'star_orders_customer.yaml', 'tpcds_date_edge.yaml', + 'school_manytomany.yaml', ]; // Same load defaults as the KC e2e/pull goldens, so the OSI golden // and the pull golden are directly comparable. diff --git a/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts index 83d51c47..a38b6b29 100644 --- a/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts @@ -54,12 +54,21 @@ const fixtures = yamlFixtures(fixturesDir); // them too and tolerate *only* missing-`expression` errors. Once PR #290 (the // sql-expressions companion aspect) regenerates these goldens with expressions // they pass with no special-casing, and any other schema drift still fails now. -function onlyMissingExpression(errors: typeof validate.errors): boolean { +function isMissingExpression(e: SchemaError): boolean { + return e.keyword === 'required' && + (e.params as {missingProperty?: string}).missingProperty === 'expression'; +} + +// One schema error, as Ajv reports it. +type SchemaError = NonNullable[number]; + +// True when every error is one a tolerance below accounts for. A fixture with +// no errors never reaches here, so an empty list is not a pass. +function onlyTolerated( + errors: typeof validate.errors, + tolerated: Array<(e: SchemaError) => boolean>): boolean { return !!errors && errors.length > 0 && - errors.every( - e => e.keyword === 'required' && - (e.params as {missingProperty?: string}).missingProperty === - 'expression'); + errors.every(e => tolerated.some(ok => ok(e))); } // `extends` (entity-level inheritance, the target of OWL rdfs:subClassOf) is a @@ -167,21 +176,18 @@ function onlyActionsExtension(errors: typeof validate.errors): boolean { // tolerate EXACTLY those three errors on a /relationships/ path and nothing // else. When upstream OSI adopts a junction-table syntax, re-vendoring the // schema makes this pass with no special-casing. -function onlyAssociationExtension(errors: typeof validate.errors): boolean { +function isAssociationSuperset(e: SchemaError): boolean { const missingOk = new Set(['from_columns', 'to_columns']); - return !!errors && errors.length > 0 && - errors.every(e => { - if (!/\/relationships\/\d+$/.test(e.instancePath)) return false; - if (e.keyword === 'required') { - return missingOk.has( - (e.params as {missingProperty?: string}).missingProperty ?? ''); - } - if (e.keyword === 'additionalProperties') { - return (e.params as {additionalProperty?: string}) - .additionalProperty === 'association'; - } - return false; - }); + if (!/\/relationships\/\d+$/.test(e.instancePath)) return false; + if (e.keyword === 'required') { + return missingOk.has( + (e.params as {missingProperty?: string}).missingProperty ?? ''); + } + if (e.keyword === 'additionalProperties') { + return (e.params as {additionalProperty?: string}).additionalProperty === + 'association'; + } + return false; } describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => { @@ -197,9 +203,15 @@ describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => if (!ok) { // A .pull.golden.yaml from an expression-free push is a known #290 gap // when its ONLY failures are missing `expression`; anything else is a - // real regression and still fails. + // real regression and still fails. A junction-backed fixture's pull + // faithfully reproduces the association superset too, so that one + // tolerates both. if (rel.endsWith('.pull.golden.yaml') && - onlyMissingExpression(validate.errors)) { + onlyTolerated( + validate.errors, + rel.startsWith('school_manytomany') ? + [isMissingExpression, isAssociationSuperset] : + [isMissingExpression])) { return; } // The OWL import goldens are purely logical models (a pre-OSI superset, @@ -227,9 +239,10 @@ describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => return; } // A junction-backed relationship is a deliberate superset too; tolerate - // exactly its three errors, and only on the fixture that carries one. - if (rel === 'school_manytomany.yaml' && - onlyAssociationExtension(validate.errors)) { + // exactly its three errors, and only on the fixture that carries one + // (and the goldens generated from it). + if (rel.startsWith('school_manytomany') && + onlyTolerated(validate.errors, [isAssociationSuperset])) { return; } const details = (validate.errors ?? []) diff --git a/toolbox/mdcode/tests/tool/init_semantic_model.test.ts b/toolbox/mdcode/tests/tool/init_semantic_model.test.ts index 463d3b07..b58e1ad3 100644 --- a/toolbox/mdcode/tests/tool/init_semantic_model.test.ts +++ b/toolbox/mdcode/tests/tool/init_semantic_model.test.ts @@ -5,7 +5,9 @@ // only entries, matching how the standard layout operates (its push creates // entries, never the entry group). The types kcmd creates rather than // references are declared in kc_custom_types.ts, which today holds the one -// action pair. These tests spy on the catalog client so no network call is +// action pair and the many-to-many association pair. The tests below assert +// over CUSTOM_TYPES rather than naming those two, so adding a third type does +// not need them rewritten. They spy on the catalog client so no network call is // made and run init // inside a temp working directory (it writes catalog.yaml + the layout dirs // relative to cwd). @@ -18,6 +20,7 @@ import * as path from 'node:path'; import {ApiResult} from '../../src/libts/gcp/api'; import {ApiContext} from '../../src/libts/gcp/context'; import {CatalogClient} from '../../src/libts/gcp/dataplex'; +import {CUSTOM_TYPES} from '../../src/libts/semantic/kc_custom_types'; import {init} from '../../src/tool/commands'; const CTX = new ApiContext('test-project', 'us', 'test-token'); @@ -45,7 +48,7 @@ beforeEach(() => { spyOn(console, 'log').mockImplementation(() => {}); spyOn(console, 'error').mockImplementation(() => {}); spyOn(console, 'warn').mockImplementation(() => {}); - // Provisioning the custom action types succeeds by default; the tests that + // Provisioning the custom types succeeds by default; the tests that // care about it re-stub these. Each create returns a long-running operation, // so `getOperation` has to answer too. spyOn(CatalogClient.prototype, 'createAspectType') @@ -100,8 +103,7 @@ describe('init --semantic-model: entry-group provisioning', () => { .toBe(true); }); - test('provisions the custom action types in the destination project', - async () => { + test('provisions every custom type in the destination project', async () => { spyOn(CatalogClient.prototype, 'createEntryGroup') .mockImplementation(async () => ok({name: 'sales-group'})); const aspectType = spyOn(CatalogClient.prototype, 'createAspectType') @@ -111,44 +113,68 @@ describe('init --semantic-model: entry-group provisioning', () => { expect(await init({semanticModel: 'proj.us.sales-group'})).toBe(0); - // Both types are custom, so they live in the destination project at - // `global` -- not beside the built-in types, and not in the entry group's - // region. + // CUSTOM_TYPES is the whole of what kcmd provisions, so init creates that + // list and nothing else -- one entry type and one aspect type per record. + const ids = CUSTOM_TYPES.map(t => t.id); for (const spy of [aspectType, entryType]) { - expect(spy).toHaveBeenCalledTimes(1); - const [project, location, typeId] = spy.mock.calls[0]; - expect(project).toBe('proj'); - expect(location).toBe('global'); - expect(typeId).toBe('semantic-action'); + expect(spy).toHaveBeenCalledTimes(ids.length); + // Every type is custom, so it lives in the destination project at + // `global` -- not beside the built-in types, and not in the entry group's + // region. + for (const call of spy.mock.calls) { + const [project, location] = call; + expect(project).toBe('proj'); + expect(location).toBe('global'); + } + expect(spy.mock.calls.map(c => c[2])).toEqual(ids); } - // The entry type requires the aspect type, so the server rejects it while - // the aspect type's create is still running. - expect(aspectType.mock.invocationCallOrder[0]) - .toBeLessThan(entryType.mock.invocationCallOrder[0]); + // An entry type requires its own aspect type, so the server rejects it + // while that aspect type's create is still running. + ids.forEach((_, i) => { + expect(aspectType.mock.invocationCallOrder[i]) + .toBeLessThan(entryType.mock.invocationCallOrder[i]); + }); }); test('waits for each type-creation operation to finish', async () => { spyOn(CatalogClient.prototype, 'createEntryGroup') .mockImplementation(async () => ok({name: 'sales-group'})); - // The aspect type is still being created when the call returns. + // Each aspect type is still being created when the call returns, and gets + // its own operation name so the polling below is per type rather than + // shared. + let created = 0; + const running = new Set(); spyOn(CatalogClient.prototype, 'createAspectType') - .mockImplementation(async () => ok({name: OP, done: false})); - let polls = 0; + .mockImplementation(async () => { + const name = `${OP}-${++created}`; + running.add(name); + return ok({name, done: false}); + }); + // An operation reports done on its SECOND poll, so a caller that does not + // wait sees it still running. + const polls = new Map(); spyOn(CatalogClient.prototype, 'getOperation') - .mockImplementation(async () => ok({name: OP, done: ++polls > 1})); - // How many times the operation had been polled when the entry type was - // created. The aspect type reports done on the second poll, so anything - // less than two means the entry type was created while its required - // aspect type was still being created, which the server rejects. - let polledBeforeEntryType = -1; + .mockImplementation(async (name: string) => { + const n = (polls.get(name) ?? 0) + 1; + polls.set(name, n); + if (n > 1) running.delete(name); + return ok({name, done: n > 1}); + }); + // How many aspect-type creates were still running each time an entry type + // was created. An entry type requires its aspect type, so the server + // rejects it while that create is in flight; anything but zero means init + // did not wait. + const stillRunning: number[] = []; spyOn(CatalogClient.prototype, 'createEntryType') .mockImplementation(async () => { - polledBeforeEntryType = polls; - return ok({name: OP}); + stillRunning.push(running.size); + return ok({name: OP, done: true}); }); expect(await init({semanticModel: 'proj.us.sales-group'})).toBe(0); - expect(polledBeforeEntryType).toBe(2); + expect(stillRunning).toEqual(CUSTOM_TYPES.map(() => 0)); + // Each aspect operation really was polled to completion, not skipped. + expect([...polls.values()]).toEqual(CUSTOM_TYPES.map(() => 2)); }); test('an already-existing aspect type is patched with the current template', @@ -166,9 +192,12 @@ describe('init --semantic-model: entry-group provisioning', () => { expect(await init({semanticModel: 'proj.us.sales-group'})).toBe(0); // A project provisioned by an earlier kcmd holds an older template, so the - // patch names the template field to bring it up to date. - expect(update).toHaveBeenCalledTimes(1); - expect(update.mock.calls[0][4]).toContain('metadata_template'); + // patch names the template field to bring it up to date -- for every custom + // type, since any of them may have grown a field. + expect(update).toHaveBeenCalledTimes(CUSTOM_TYPES.length); + for (const call of update.mock.calls) { + expect(call[4]).toContain('metadata_template'); + } }); test('a rejected template patch leaves the existing type and finishes init', @@ -185,6 +214,7 @@ describe('init --semantic-model: entry-group provisioning', () => { async () => err(400, 'backwards-incompatible template change')); const entryType = spyOn(CatalogClient.prototype, 'createEntryType') .mockImplementation(async () => err(409, 'exists')); + // One warning per type whose patch was refused. const warned: string[] = []; spyOn(console, 'warn').mockImplementation((...args: any[]) => { warned.push(args.join(' ')); @@ -194,27 +224,28 @@ describe('init --semantic-model: entry-group provisioning', () => { // reports the refusal and carries on rather than abandoning a workspace // whose entry group it has already created. expect(await init({semanticModel: 'proj.us.sales-group'})).toBe(0); - expect(entryType).toHaveBeenCalledTimes(1); + expect(entryType).toHaveBeenCalledTimes(CUSTOM_TYPES.length); expect(fs.existsSync(path.join('catalog', 'EntryGroups', 'sales-group'))) .toBe(true); expect(warned.some(m => m.includes('unchanged'))).toBe(true); }); - test('init survives lacking permission to create the action types', + test('init survives lacking permission to create the custom types', async () => { spyOn(CatalogClient.prototype, 'createEntryGroup') .mockImplementation(async () => ok({name: 'sales-group'})); spyOn(CatalogClient.prototype, 'createAspectType') .mockImplementation(async () => err(403, 'permission denied')); - // Actions are one optional construct: a caller who will never declare one - // must still be able to init, push and pull. + // Every custom type backs an optional construct -- an action, a + // many-to-many relationship -- so a caller who will never declare one must + // still be able to init, push and pull. expect(await init({semanticModel: 'proj.us.sales-group'})).toBe(0); expect(fs.existsSync(path.join('catalog', 'EntryGroups', 'sales-group'))) .toBe(true); }); - test('a fatal action-type error fails init', async () => { + test('a fatal custom-type error fails init', async () => { spyOn(CatalogClient.prototype, 'createEntryGroup') .mockImplementation(async () => ok({name: 'sales-group'})); spyOn(CatalogClient.prototype, 'createAspectType') From 146a4bf27e306afb81d4c73f5fa68e25e942d5fa Mon Sep 17 00:00:00 2001 From: Bei Li Date: Tue, 8 Sep 2026 20:48:48 +0000 Subject: [PATCH 3/3] mdcode: rename the M:N construct to `through` on the relationship `association` named a concept the graph stores does not have. BigQuery Graph and Spanner Graph both model one thing here -- a relationship with a name, two endpoints, join columns, and properties -- and whether the edge is carried by a foreign key or by a table of pairs changes only where those columns live. So the authored form flattens: a relationship gains `through` (the table it runs through), plus the `keys` and `fields` that table makes possible. `from_columns`/`to_columns` keep their meaning; `through` says which table they are on. The Knowledge Catalog type follows the same reasoning, and widens. `semantic-association` becomes `semantic-relationship`, and EVERY relationship publishes as one such entry, not just a many-to-many one -- the aspect already fit the foreign-key case with no new fields. A foreign-key relationship also keeps its `schema-join` link, for the Dataplex surfaces that read links: the entry is the fidelity record, the link the graph-shaped projection. Pull prefers the entries and falls back to the links for a catalog written by an older kcmd. Widening closes three losses the fidelity doc previously recorded: - a relationship's name came back lowercased and hyphenated from the link id; the entry carries it verbatim - relationship-level ai_context.instructions had no catalog home - a purely logical, column-less relationship was skipped entirely Also fixes a defect the flattening exposed: pruneUnavailable keyed a relationship's join columns to its endpoint entities, which is wrong for a through-edge -- those columns are on the through table, so a same-named unbound field on an endpoint would have dropped the edge. --- toolbox/mdcode/docs/semantic-model/README.md | 12 +- .../mdcode/docs/semantic-model/fidelity.md | 88 +++--- .../mdcode/docs/semantic-model/model_spec.md | 82 +++--- .../mdcode/docs/semantic-model/profiles.md | 2 +- .../mdcode/docs/semantic-model/reference.md | 56 ++-- toolbox/mdcode/src/libts/semantic/bigquery.ts | 44 +-- .../semantic/deploy_knowledge_catalog.ts | 10 +- toolbox/mdcode/src/libts/semantic/ir.ts | 59 ++-- .../mdcode/src/libts/semantic/kc_converter.ts | 41 +-- .../src/libts/semantic/kc_custom_types.ts | 92 ++++--- ...kc_associations.ts => kc_relationships.ts} | 246 +++++++++-------- .../src/libts/semantic/knowledge_catalog.ts | 108 ++++---- toolbox/mdcode/src/libts/semantic/loader.ts | 190 +++++++------ .../src/libts/semantic/osi_converter.ts | 54 ++-- toolbox/mdcode/src/libts/semantic/pull_kc.ts | 12 +- .../src/libts/semantic/resolve_profiles.ts | 12 +- toolbox/mdcode/src/libts/semantic/spanner.ts | 37 ++- .../mdcode/src/libts/semantic/transpile.ts | 4 +- toolbox/mdcode/src/libts/semantic/validate.ts | 8 +- .../tests/libts/semantic/bigquery.test.ts | 35 ++- .../semantic/deploy_knowledge_catalog.test.ts | 5 +- ...l_manytomany.knowledge_catalog.golden.json | 10 +- .../school_manytomany.osi.golden.yaml | 29 +- .../school_manytomany.pull.golden.yaml | 31 ++- .../semantic/fixtures/school_manytomany.yaml | 39 ++- ...ers_customer.knowledge_catalog.golden.json | 27 +- .../star_orders_customer.pull.golden.yaml | 2 +- ...ds_date_edge.knowledge_catalog.golden.json | 98 ++++++- .../fixtures/tpcds_date_edge.pull.golden.yaml | 8 +- .../tests/libts/semantic/kc_converter.test.ts | 236 +++++++++------- .../semantic/knowledge_catalog.e2e.test.ts | 6 +- .../libts/semantic/knowledge_catalog.test.ts | 251 ++++++++---------- .../tests/libts/semantic/loader.test.ts | 107 ++++---- .../libts/semantic/osi_converter.test.ts | 40 ++- .../tests/libts/semantic/osi_schema.test.ts | 50 ++-- .../libts/semantic/resolve_profiles.test.ts | 15 ++ .../tests/libts/semantic/spanner.test.ts | 31 +-- .../tests/libts/semantic/transpile.test.ts | 26 +- .../tests/tool/init_semantic_model.test.ts | 4 +- 39 files changed, 1171 insertions(+), 1036 deletions(-) rename toolbox/mdcode/src/libts/semantic/{kc_associations.ts => kc_relationships.ts} (50%) diff --git a/toolbox/mdcode/docs/semantic-model/README.md b/toolbox/mdcode/docs/semantic-model/README.md index 393c9447..bf420959 100644 --- a/toolbox/mdcode/docs/semantic-model/README.md +++ b/toolbox/mdcode/docs/semantic-model/README.md @@ -115,12 +115,12 @@ metric's `expression` may be a bare formula over the logical fields or the fulle per-dialect form. `entities` may also be written `datasets` (the two are interchangeable under the `/google` version). A relationship that pairs many rows on each side — a student takes many courses, -a course has many students — is written with an `association` block instead of -`from_columns` / `to_columns`. The block names the junction table that holds the -pairs, the columns that reach each side, and any fields the pairing itself -carries (an enrollment's grade, say). Both graphs deploy it as an edge table over -the junction; Knowledge Catalog stores it as a `semantic-association` entry. See -[Model spec §2.2.1](model_spec.md#221-many-to-many-association). +a course has many students — adds `through`, naming the table that holds the +pairs. `from_columns` and `to_columns` then reach each side from that table +rather than from the endpoints, and the relationship may carry a key of its own +plus any fields the pairing itself has (an enrollment's grade, say). Both graphs +deploy it as an edge table over that table. See +[Model spec §2.2.1](model_spec.md#221-many-to-many-through). Entities can **extend** other entities (`extends: [Parent]`); push flattens the supertype's fields down and expresses the hierarchy as graph labels, so a query diff --git a/toolbox/mdcode/docs/semantic-model/fidelity.md b/toolbox/mdcode/docs/semantic-model/fidelity.md index 35e752bf..2dab387f 100644 --- a/toolbox/mdcode/docs/semantic-model/fidelity.md +++ b/toolbox/mdcode/docs/semantic-model/fidelity.md @@ -32,8 +32,8 @@ agree on every structural row and differ only where a Spanner target has no | Primary key | `schema.primaryKey` | ✓ | `KEY(...)` on the node table | `KEY(...)` on the node table | | Unique keys | `schema.uniqueConstraints` | ✓ | — dropped (only PK emitted) | — dropped (only PK emitted) | | Metric | `semantic-metric` entry | name, entity, description, instructions, type⁵ | `MEASURE`⁴ | — dropped (no `MEASURE`) | -| Relationship (1:1 / 1:N) | `schema-join` link | ✓ (name normalized⁶) | `EDGE TABLE` | `EDGE TABLE` | -| Relationship (M:N / `association`) | `semantic-association` entry¹³ | ✓¹³ | `EDGE TABLE` (via junction table) | `EDGE TABLE` (via junction table) | +| Relationship (foreign key) | `semantic-relationship` entry⁶ | ✓ | `EDGE TABLE` | `EDGE TABLE` | +| Relationship (`through` a table of pairs) | `semantic-relationship` entry⁶ | ✓¹³ | `EDGE TABLE` over that table | `EDGE TABLE` over that table | | Entity `extends` | — not modelled | — | `LABEL` clauses + flattened fields | `LABEL` clauses + flattened fields | | Action | `semantic-action` entry¹² | ✓¹² | — not represented (write-side) | — not represented (write-side) | | `description` (entity / metric / field / relationship) | entry description / aspect | ✓ | `OPTIONS(description)` | — dropped | @@ -63,17 +63,18 @@ agree on every structural row and differ only where a Spanner target has no 5. **Metric type.** A metric's expression is gated behind `--emit-expressions`; its data type round-trips only for a concrete type (e.g. `Decimal`) — an untyped, `String`, or `Opaque` metric comes back un-typed. -6. **Relationship name.** A one-to-many relationship's name comes back - lowercased/hyphenated (`Places Order` → `places-order`) — the catalog stores - the name only in the link id. See - [Writer-side follow-up](#writer-side-follow-up). A many-to-many relationship - is stored as an entry, not a link, so its name comes back verbatim. +6. **Relationship storage.** Every relationship is stored as one + `semantic-relationship` entry under the model entry, holding both endpoints, + the join columns, and the relationship's `instructions`. A relationship + carried by a foreign key *also* gets a `schema-join` link, so Dataplex + surfaces that read joins still see it; the entry is the fidelity record and + the link the graph-shaped projection. Because the name rides the entry, it + comes back verbatim — a link could only ever return the lowercased, + hyphenated form of its id. 7. **Guidelines aspect.** The `guidelines` aspect exists only for the model, - entities, and metrics — not fields or relationships, so field- and - relationship-level `ai_context.instructions` has no Knowledge Catalog home (a - relationship's instructions still reach BigQuery, folded into the edge's - `OPTIONS(description)`). The one exception is a many-to-many relationship, - whose `semantic-association` aspect carries its `instructions` — that aspect + entities, and metrics — not fields, so field-level + `ai_context.instructions` has no Knowledge Catalog home. A relationship's + instructions ride its own `semantic-relationship` aspect instead: that aspect type is ours, so it has a field for them. 8. **Model-level metadata.** Neither graph has a home for statement-level metadata — BigQuery silently drops graph-statement `OPTIONS`, and Spanner @@ -90,7 +91,8 @@ agree on every structural row and differ only where a Spanner target has no 10. **Logical (unbound) model.** A model with no bindings still publishes to Knowledge Catalog: each entity's `source` is recorded empty (`resources: []`) because there is no table behind it, and a relationship that carries no join - columns is skipped with a warning. When the same push also deploys a graph, + columns publishes as an entry with none (it gets no `schema-join` link, which + requires a column pair). When the same push also deploys a graph, the catalog entries are first pruned to what the graph binds — see [To Knowledge Catalog](#to-knowledge-catalog). 11. **Vendor-dialect fallback.** What you author is the `expression.dialects[]` @@ -107,22 +109,21 @@ agree on every structural row and differ only where a Spanner target has no scope: an action's `precondition` and `affects` are not modelled, so nothing about them is stored either way. See [Modeling write operations](actions.md). -13. **Many-to-many relationships.** The `schema-join` link holds exactly one - source/target column pair, so it cannot describe an edge that runs through a - junction table. A many-to-many relationship is published instead as one - `semantic-association` entry under the model entry, holding the two entities - it pairs, the junction table, and the junction's keys, join columns, and - fields. The whole `association` block round-trips — including the edge's own - fields and their expressions, which are stored unconditionally (the aspect - type is ours, so it has fields for them; the `--emit-expressions` gate exists - for the published system templates that do not). See - [Model spec §2.2.1](model_spec.md#221-many-to-many-association). +13. **Relationships through a table of pairs.** A relationship with a `through` + table gets no `schema-join` link: that link holds exactly one source/target + column pair, which cannot describe an edge running through a third table. Its + entry carries the rest — the `through` table, its key, and the fields of the + pairing — so the whole relationship round-trips, the edge's own field + expressions included. Those are stored unconditionally: the aspect type is + ours, so it has fields for them, where the `--emit-expressions` gate exists + for the published system templates that do not. See + [Model spec §2.2.1](model_spec.md#221-many-to-many-through). ## To Knowledge Catalog The catalog holds metadata rather than a full copy of your model. Every resource type it uses is a built-in system type under `dataplex-types/global`, apart from -the custom `semantic-association` and `semantic-action` pairs that `kcmd init` +the custom `semantic-relationship` and `semantic-action` pairs that `kcmd init` provisions — push references types, it never creates them (see [Reference → What gets created in Knowledge Catalog](reference.md#what-gets-created-in-knowledge-catalog)). @@ -137,7 +138,7 @@ authored model. A logical model still produces complete entries. Each entity's `source` is recorded empty (`resources: []`) because there is no table behind it, and a -relationship that carries no join columns is skipped with a warning. +relationship that carries no join columns publishes as an entry with none. By default the catalog does **not** store the SQL expressions: the published system-type templates do not yet carry a per-field `semantics` block or a @@ -150,14 +151,16 @@ SQL (`importedExpression` — for example the MAQL or Snowflake form a metric wa imported from). Those stay in your authored document; the vendor SQL and expressions are still used when generating graph SQL. -**Many-to-many relationships** get an entry rather than a link. A `schema-join` -link holds one source/target column pair, which cannot describe an edge that -runs through a junction table, and Knowledge Catalog has no custom *link* types -— only custom entry and aspect types. So each many-to-many relationship becomes -a `semantic-association` entry under the model entry, carrying the two entities -it pairs and the junction table with its keys, join columns, and fields. The -whole block round-trips through `pull`, name included. The entry type is custom, -so `kcmd init` creates it; a model with no many-to-many relationship never needs +**Relationships** get an entry rather than a link. A `schema-join` link holds one +source/target column pair, which cannot describe an edge running through a table +of pairs, and it has no field for the relationship's name; Knowledge Catalog has +no custom *link* types either — only custom entry and aspect types. So each +relationship becomes a `semantic-relationship` entry under the model entry, +carrying the two entities it pairs, the join columns, and, when there is one, the +`through` table with its key and fields. All of it round-trips through `pull`, +name included. A relationship carried by a foreign key also keeps its +`schema-join` link, for the Dataplex surfaces that read joins. The entry type is +custom, so `kcmd init` creates it; a model with no relationships never needs it. **Actions** follow the same one-entry-per-element rule as everything else: each @@ -230,10 +233,6 @@ returns that view. Two things about *how* it comes back: **Normalized** — the content survives, the form changes: -- A one-to-many relationship's *name* comes back lowercased/hyphenated - (`Places Order` → `places-order`); the catalog stores the name only in the link - id. See [Writer-side follow-up](#writer-side-follow-up). A many-to-many - relationship is stored as an entry instead, so its name comes back verbatim. - Field types round-trip except two collapses: a field authored with no type comes back as `Opaque`, and a field authored as `String` comes back un-typed (both store `dataType STRING`, kept distinct by a field's `metadataType` — see @@ -252,21 +251,6 @@ returns that view. Two things about *how* it comes back: pulled document as a faithful copy of the catalog metadata rather than of the authored model, and keep the authored document as the source of truth. -## Writer-side follow-up - -One reduction above is a limit of what push currently *writes* rather than of -what pull can recover. It is recorded here as a write-side follow-up; the reader -(pull) already returns everything the catalog holds. - -- **Relationship names** (one-to-many only). The `schema-join` aspect type's - `metadataTemplate` has no field for the relationship name, so push cannot store it and pull recovers - it from the link id — which is lowercased and hyphenated (the entry-link id - format forbids the original casing/underscores). Returning the name verbatim - requires adding a name field to the built-in `schema-join` aspect type in - Knowledge Catalog (server-side), after which the client write/read is trivial; - it is the same class of gap as the `semantics` field that gates - `--emit-expressions`. - (A non-canonical deployment target is **not** a pull gap: push rejects it at the validation gate before any leg runs, so it is never written — see [Validation](reference.md#validation).) diff --git a/toolbox/mdcode/docs/semantic-model/model_spec.md b/toolbox/mdcode/docs/semantic-model/model_spec.md index 13e56812..eb112203 100644 --- a/toolbox/mdcode/docs/semantic-model/model_spec.md +++ b/toolbox/mdcode/docs/semantic-model/model_spec.md @@ -199,17 +199,19 @@ A relationship is a directed edge between two datasets. | `name` | string | required | | `from` | string | required; a declared dataset name | | `to` | string | required; a declared dataset name | -| `from_columns` | list of strings | join key on `from` | -| `to_columns` | list of strings | join key on `to` | -| `association` | [association](#221-many-to-many-association) | `/google` only; a many-to-many edge | +| `from_columns` | list of strings | join key reaching `from` | +| `to_columns` | list of strings | join key reaching `to` | +| `through` | string | `/google` only; a table holding the pairs · [§2.2.1](#221-many-to-many-through) | +| `keys` | list of strings | `/google` only; requires `through` | +| `fields` | list of [field](#211-field) | `/google` only; requires `through` | | `description` | string | | | `ai_context` | [ai_context](#24-ai_context) | | | `custom_extensions` | list | [§6](#6-the-extension-mechanism) | `from_columns` and `to_columns` are the edge's join keys. They MUST be given together (a bound edge) or both omitted (a logical edge); one without the other is -rejected. When both are given they MUST have equal length. `from` and `to` MUST -name datasets declared in the same model. +rejected. Without `through` they MUST have equal length, because they pair up +positionally. `from` and `to` MUST name datasets declared in the same model. Ossie **requires** both join-column lists; allowing both to be omitted — a logical edge with no join keys — is a `kcmd` **relaxation** ([§4](#4-narrowings-and-relaxations)) @@ -218,44 +220,46 @@ requires them (see [§4](#4-narrowings-and-relaxations)). Unlike `source` and fi join columns are declared on the logical model and are not profile-swappable ([§7](#7-the-binding-layer)). -#### 2.2.1. Many-to-many (`association`) +#### 2.2.1. Many-to-many (`through`) -A relationship whose two sides each match many of the other cannot be a foreign -key: a foreign-key column holds one value, so it references at most one row. The -pairs live in a table of their own — one row per pair — and the edge is declared -with an `association` block instead of the relationship's own join columns. - -| Key | Type | Rule | -|---|---|---| -| `source` | string | required; the table holding the pairs | -| `from_columns` | list of strings | required; columns on that table referencing `from` | -| `to_columns` | list of strings | required; columns on that table referencing `to` | -| `keys` | list of strings | the pairing's own key; defaults to the two column lists combined | -| `fields` | list of [field](#211-field) | properties of the pairing itself | +A relationship whose two sides each match many of the other cannot be carried by +a foreign key: a foreign-key column holds one value, so it references at most one +row. The pairs live in a table of their own — one row per pair — and the +relationship names it with `through`. ```yaml relationships: - name: enrollment from: students to: courses - association: - source: analytics.school.enrollment - keys: [enrollment_id] - from_columns: [student_id] - to_columns: [course_id] - fields: - - name: grade - expression: enrollment.grade + through: analytics.school.enrollment + keys: [enrollment_id] + from_columns: [student_id] + to_columns: [course_id] + fields: + - name: grade + expression: enrollment.grade ``` -The columns named inside the block are columns of the **pairing table**, not of -either endpoint, and each list references the corresponding endpoint's declared -`primary_key`. A relationship MUST NOT carry both an `association` and its own -`from_columns`/`to_columns`: an edge is one shape or the other. `fields` are -properties of the pairing rather than of either side — a grade belongs to the -enrollment, not to the student or the course. +`through` changes where `from_columns` and `to_columns` live, not what they mean. +Without it they are on the two endpoints' own tables; with it both are on the +table named by `through`, one list reaching each endpoint's declared +`primary_key`. Because the two lists no longer pair up with each other, they need +not have equal length. Both are required when `through` is given: without them +nothing says which pairs the table holds. + +`keys` is the edge's own key, on the `through` table. It defaults to the two +column lists combined and deduplicated, which is unique whenever a pair appears +at most once; give it explicitly for a surrogate key, or when a pair may +legitimately repeat (an enrollment per term). + +`fields` are properties of the pairing rather than of either endpoint — a grade +belongs to the enrollment, not to the student or the course. + +`keys` and `fields` both require `through`. An edge carried by a foreign key has +no table of its own, so it has nowhere to put a key or a property. -`association` is a native key of the extended profile +`through`, `keys`, and `fields` are native keys of the extended profile ([§5](#5-extensions)); vanilla Ossie has no syntax for a pairing table. ### 2.3. Metric @@ -341,7 +345,7 @@ extension, [§5](#5-extensions)), or *rejected* / *not authorable* (excluded). | `abstract` | — | added | supertype with no table; `/google` only · [§5](#5-extensions) | | relationship `name`, `from`, `to` | defined | same | [§2.2](#22-relationship) | | relationship `from_columns` / `to_columns` | required | optional | model before binding; none = logical edge · [§4.2](#42-relaxations-looser-than-ossie) | -| relationship M:N (`association`) | — | added | pairing table + its own key and fields; `/google` only · [§2.2.1](#221-many-to-many-association), [§5](#5-extensions) | +| relationship M:N (`through`) | — | added | pairing table + its own key and fields; `/google` only · [§2.2.1](#221-many-to-many-through), [§5](#5-extensions) | | `metrics`, metric `expression` | required | same; graph-bound stricter | a graph measure binds one node and aggregate · [§4.1](#41-narrowings-stricter-than-ossie) | | `expression.dialects` | closed enum | any dialect string | tolerate imported / newer input · [§4.2](#42-relaxations-looser-than-ossie) | | field `expression` (column binding) | required | optional | model before binding; unbound is pruned · [§4.2](#42-relaxations-looser-than-ossie), [§7](#7-the-binding-layer) | @@ -390,8 +394,8 @@ Each rule and its reason: - **A graph-bound relationship MUST have its join columns bound.** For any graph target, a relationship MUST supply both `from_columns` and `to_columns` before - deploy — on the relationship itself, or, for a many-to-many edge, inside its - `association` block. *Why:* the edge table needs both keys. + deploy, whether they sit on the endpoints' own tables or on a `through` table. + *Why:* the edge table needs both keys. - **Unknown keys are rejected.** Every object is validated closed: an unrecognized sibling key is a hard load error, not silently dropped. Combined with the version @@ -490,11 +494,11 @@ reads the document ([§6](#6-the-extension-mechanism)). bindings, so one logical model serves several stores. Not part of the Ossie document; a `kcmd`-specific file alongside it ([§7](#7-the-binding-layer)). -- **`association` (extended profile only).** A many-to-many relationship, backed +- **`through` (extended profile only).** A many-to-many relationship, backed by a table of pairs with its own key and its own properties. Ossie's relationship is a foreign key only, and the carrier cannot express one either: a `custom_extensions` block holds opaque data, and this edge has to be read by - the graph generators. Grammar in [§2.2.1](#221-many-to-many-association). + the graph generators. Grammar in [§2.2.1](#221-many-to-many-through). ## 6. The extension mechanism @@ -643,7 +647,7 @@ The full merge behavior and worked examples are in and constructs with no vanilla form (inheritance, the `entities` spelling) are simply unavailable there — a model that needs them uses `0.2.0.dev0/google`. -- **Extensions are additive.** `association` and `actions` were both added to +- **Extensions are additive.** `through` and `actions` were both added to the extended profile after `0.2.0.dev0/google` was first published, each as a new optional key. A document that used neither is unaffected, which is the shape any further extension takes. diff --git a/toolbox/mdcode/docs/semantic-model/profiles.md b/toolbox/mdcode/docs/semantic-model/profiles.md index d798d3ec..d6b3b87e 100644 --- a/toolbox/mdcode/docs/semantic-model/profiles.md +++ b/toolbox/mdcode/docs/semantic-model/profiles.md @@ -82,7 +82,7 @@ each field reads. A profile sets binding and leaves declaration alone. | an entity's `source` (its store URI) | which entities, fields, relationships, or metrics exist, and what each means | | a field's column (its `expression`, a bare column reference) | a field's `label`, `description`, `dimension`, `datatype` | | whether a field is bound at all under this profile | the grain (`primary_key` / `unique_keys`) and graph shape (`from`/`to`, `from_columns`/`to_columns`) | -| the deployment target | a field `expression` that is arbitrary SQL, which changes the computation; any `metric` definition; any `ai_context` / synonyms; a relationship or its junction `source` | +| the deployment target | a field `expression` that is arbitrary SQL, which changes the computation; any `metric` definition; any `ai_context` / synonyms; a relationship or its `through` table | An element's `name` is not overridden — it is the key that pairs a profile element with the model element it binds. The grain and the join columns name diff --git a/toolbox/mdcode/docs/semantic-model/reference.md b/toolbox/mdcode/docs/semantic-model/reference.md index f3b37fc9..88ec8c33 100644 --- a/toolbox/mdcode/docs/semantic-model/reference.md +++ b/toolbox/mdcode/docs/semantic-model/reference.md @@ -259,7 +259,7 @@ are **not** probed before deploy — the live pre-flight is BigQuery-only (see ## What gets created in Knowledge Catalog Each element of your model maps to one catalog resource. Every resource type -below except `semantic-association` and `semantic-action` is a built-in system +below except `semantic-relationship` and `semantic-action` is a built-in system type under `dataplex-types/global` — push references them, it never creates them. Those two are custom, and `kcmd init --semantic-model` creates them in your own project at `global`; push still writes only entries. @@ -273,26 +273,31 @@ your own project at `global`; push still writes only entries. | Model | `semantic-model` | entry — anchor / parent of the rest | `` | | Entity | `semantic-entity` (+ built-in `schema` aspect) | entry | `.entities.` | | Metric | `semantic-metric` | entry | `.metrics.` | -| Relationship (1:1 / 1:N) | `schema-join` | entry link between the two entity entries | derived from the model and relationship names | -| Relationship (M:N) | `semantic-association` (custom type) | entry | `.associations.` | +| Relationship | `semantic-relationship` (custom type) | entry | `.relationships.` | +| Relationship, foreign-key only | `schema-join` (in addition to the entry above) | entry link between the two entity entries | derived from the model and relationship names | | Action | `semantic-action` (custom type) | entry | `.actions.` | An entity entry carries its columns in the `schema` aspect (name, data type, description, and any `label` per field), plus the entity's keys and unique keys -(`primaryKey` / `uniqueConstraints`); a `schema-join` link carries the -relationship detail — the paired columns and foreign-key direction — in its -aspect. Any element with `ai_context.instructions` (the model, an entity, or a -metric) also gets a built-in `guidelines` aspect holding that text. - -A **many-to-many** relationship gets an entry rather than a link: a -`schema-join` link holds exactly one source/target column pair, so it cannot -describe an edge that runs through a junction table, and Dataplex has no custom -*link* types — only custom entry and aspect types. Its `semantic-association` -aspect holds the two entities it pairs, the junction table, and the junction's -keys, join columns, and fields, along with the relationship's -`ai_context.instructions`. The whole `association` block round-trips through -`pull`, the relationship name included. See -[Model spec §2.2.1](model_spec.md#221-many-to-many-association). +(`primaryKey` / `uniqueConstraints`). Any element with `ai_context.instructions` +(the model, an entity, or a metric) also gets a built-in `guidelines` aspect +holding that text. + +Every relationship gets an **entry** of its own. A `schema-join` link cannot be +the relationship's home: it holds exactly one source/target column pair, so it +cannot describe an edge running through a table of pairs, and it has no field for +the relationship's name — a pull can only recover the slug in its id. Dataplex +has no custom *link* types either, only custom entry and aspect types. The +`semantic-relationship` aspect holds the two entities the relationship pairs, the +join columns, the `through` table with its key and fields when there is one, and +the relationship's `ai_context.instructions`. All of it round-trips through +`pull`, the relationship name verbatim. + +A relationship carried by a **foreign key** also gets its `schema-join` link, so +Dataplex surfaces that read joins still see it. The entry is the fidelity record; +the link is the graph-shaped projection. Pull reads the entries and ignores the +links, falling back to them only for a catalog written before relationships had +entries. See [Model spec §2.2.1](model_spec.md#221-many-to-many-through). An **action** entry carries its executor and its typed parameters in a `semantic-action` aspect, along with the action's `ai_context.instructions`. @@ -413,10 +418,10 @@ and each aspect type attached, so a push needs, on the destination entry group: `semantic-entity`, and `semantic-metric` aspect types the push attaches — i.e. `dataplex.entryGroups.useSemanticModelAspect`, `useSemanticEntityAspect`, and `useSemanticMetricAspect` -* `dataplex.aspectTypes.use` on the `semantic-association` aspect type, when the - model has many-to-many relationships, and on the `semantic-action` aspect - type, when it declares actions — those types are custom rather than built-in, - so they are authorized on the type resource instead of through an entry-group +* `dataplex.aspectTypes.use` on the `semantic-relationship` aspect type, when the + model has relationships, and on the `semantic-action` aspect type, when it + declares actions — those types are custom rather than built-in, so they are + authorized on the type resource instead of through an entry-group use-permission > The `schema` / `guidelines` / `schema-join` use-permissions follow Dataplex's @@ -431,15 +436,14 @@ needs more than push does, in the destination project: * `dataplex.entryGroups.create` — the destination entry group * `dataplex.aspectTypes.create` / `dataplex.aspectTypes.update` and - `dataplex.entryTypes.create` — the custom `semantic-association` and + `dataplex.entryTypes.create` — the custom `semantic-relationship` and `semantic-action` pairs. Init patches an aspect type that is already there, so a project set up by an older `kcmd` picks up template additions; an entry type that is already there is left alone. -Only the entry-group permission is required. Many-to-many relationships and -actions are both optional constructs, so init reports a refusal to create their -types as a warning and carries on; a model that uses neither still pushes and -pulls. Any other failure to create a type stops init, rather than leaving a +Only the entry-group permission is required. Relationships and actions are both +optional constructs, so init reports a refusal to create their types as a warning +and carries on; a model that uses neither still pushes and pulls. Any other failure to create a type stops init, rather than leaving a later push to hit an opaque parsing error. `kcmd pull` needs read access to the same entry group instead — to list its diff --git a/toolbox/mdcode/src/libts/semantic/bigquery.ts b/toolbox/mdcode/src/libts/semantic/bigquery.ts index 9c8b721f..7c0140d4 100644 --- a/toolbox/mdcode/src/libts/semantic/bigquery.ts +++ b/toolbox/mdcode/src/libts/semantic/bigquery.ts @@ -19,7 +19,7 @@ // See: https://docs.cloud.google.com/bigquery/docs/graph-measures // -import {AiContext, Association, Entity, Field, fieldBinding, isTimeDimension, Metric, Relationship, SemanticModel,} from './ir'; +import {AiContext, Entity, Field, fieldBinding, isTimeDimension, Metric, Relationship, SemanticModel,} from './ir'; import {resolveInheritance} from './resolve_inheritance'; import {referencedEntityNames, stripQualifier} from './sql_expr_utils'; import {isSimpleIdentifier, quoteIfReserved} from './sql_identifiers'; @@ -827,11 +827,10 @@ function physicalColumns( function renderEdgeTable( rel: Relationship, entitiesByName: Map, opts: GenerateOptions, warnings: string[]): string { - // A many-to-many relationship is backed by its own association table rather - // than a source entity's foreign key; render it from that block. - if (rel.association) { - return renderAssociationEdge( - rel, rel.association, entitiesByName, opts, warnings); + // A many-to-many relationship runs through a table of its own rather than + // over a source entity's foreign key; render it from that table. + if (rel.through) { + return renderThroughEdge(rel, rel.through, entitiesByName, opts, warnings); } // A relationship is a direct foreign key: the SOURCE entity's own base table // backs the edge (one edge row per source row). Its FK columns @@ -890,19 +889,19 @@ function renderEdgeTable( } -// Renders a many-to-many edge backed by an association (junction) table. Unlike -// a direct FK, the edge has its OWN backing table and KEY, each endpoint's -// SOURCE/DESTINATION KEY names the junction columns referencing that entity's -// declared key, and the junction's own `fields` become edge PROPERTIES. -function renderAssociationEdge( - rel: Relationship, assoc: Association, entitiesByName: Map, +// Renders a many-to-many edge, which runs through a table of its own. Unlike a +// direct FK, the edge has its OWN backing table and KEY, each endpoint's +// SOURCE/DESTINATION KEY names the columns on that table referencing the +// entity's declared key, and the edge's own `fields` become PROPERTIES. +function renderThroughEdge( + rel: Relationship, through: string, entitiesByName: Map, opts: GenerateOptions, warnings: string[]): string { - const backing = qualifyTable( - assoc.dataSource, opts, warnings, `relationship '${rel.name}'`); - if (!assoc.keys?.length) { + const backing = + qualifyTable(through, opts, warnings, `relationship '${rel.name}'`); + if (!rel.keys?.length) { warnings.push( - `relationship '${rel.name}': association table has no KEY; the edge ` + - `table will be invalid (an edge requires a KEY)`); + `relationship '${rel.name}': the table it runs through has no KEY; ` + + `the edge table will be invalid (an edge requires a KEY)`); } // The REFERENCES target is each endpoint entity's declared key; fall back to @@ -920,16 +919,17 @@ function renderAssociationEdge( const lines = [ line(1, `${backing} AS ${quoteIfReserved(rel.name)}`), - line(2, `KEY(${assoc.keys.map(quoteIfReserved).join(', ')})`), + line(2, `KEY(${(rel.keys ?? []).map(quoteIfReserved).join(', ')})`), line( 2, - `SOURCE KEY(${assoc.sourceColumns.map(quoteIfReserved).join(', ')}) ` + + `SOURCE KEY(${ + rel.source.columns.map(quoteIfReserved).join(', ')}) ` + `REFERENCES ${quoteIfReserved(rel.source.entity)}(${ refColumns(rel.source)})`), line( 2, `DESTINATION KEY(${ - assoc.destinationColumns.map(quoteIfReserved).join(', ')}) ` + + rel.destination.columns.map(quoteIfReserved).join(', ')}) ` + `REFERENCES ${quoteIfReserved(rel.destination.entity)}(${ refColumns(rel.destination)})`), ]; @@ -942,9 +942,9 @@ function renderAssociationEdge( rel.aiContext?.synonyms); if (labelOpts) lines.push(line(2, labelOpts)); - // The junction's own non-key fields are the edge's properties. + // The through table's own non-key fields are the edge's properties. const properties = - (assoc.fields ?? []).map(f => renderFieldProperty(f, rel.name)); + (rel.fields ?? []).map(f => renderFieldProperty(f, rel.name)); if (properties.length) lines.push(propertiesBlock(properties)); return lines.join('\n'); diff --git a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts index ca8a0869..96fa9013 100644 --- a/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/deploy_knowledge_catalog.ts @@ -22,11 +22,11 @@ // * Reconcile deletions: an entity or metric removed from a still-present // model leaves an orphaned entry under its anchor; after writing, delete // any entry this push owns (by entry-id prefix) that was not re-emitted. -// * Relationship edges are published as schema-join entry links between the -// two entity entries, written after that model's entries (both endpoints -// must exist first). A re-push upserts the link's aspect; a many-to-many -// (association) edge is not published yet (the emitter warns and skips it). -// The caller additionally needs `dataplex.entryGroups.useSchemaJoinEntryLink` +// * Every relationship is published as an entry among that model's entries. +// A foreign-key edge is ALSO published as a schema-join entry link between +// the two entity entries, written after those entries (both endpoints must +// exist first); a re-push upserts the link's aspect. For the links the +// caller additionally needs `dataplex.entryGroups.useSchemaJoinEntryLink` // and `useSchemaJoinAspect` on the destination entry group. // // This is a library module: it emits no console output. Warnings and the diff --git a/toolbox/mdcode/src/libts/semantic/ir.ts b/toolbox/mdcode/src/libts/semantic/ir.ts index 73070876..e72c1e15 100644 --- a/toolbox/mdcode/src/libts/semantic/ir.ts +++ b/toolbox/mdcode/src/libts/semantic/ir.ts @@ -219,48 +219,59 @@ export interface Relationship { name: string; source: RelationshipEnd; destination: RelationshipEnd; - // When present, this edge is a many-to-many backed by a junction table rather - // than a direct foreign key on the source entity. See Association. - association?: Association; + // When present, the edge runs THROUGH this table -- one holding one row + // per pair -- rather than over a foreign key on the source entity's own table. + // This is what makes the edge many-to-many; see below for why, and note that + // it changes what the endpoints' `columns` mean. + through?: string; + // The edge's own key, on the `through` table. Only a through-edge has one: a + // foreign-key edge is keyed by its source entity's key, which the generators + // look up from the entity rather than carry here. + keys?: string[]; + // Properties of the pairing itself -- an enrollment's grade. Only a + // through-edge has any: a foreign-key edge has no table of its own to hold + // them. + fields?: Field[]; description?: string; aiContext?: AiContext; customExtensions?: CustomExtension[]; } /** - * One endpoint of a relationship: the entity it attaches to and the columns on - * that entity's own table that participate in the join. + * One endpoint of a relationship: the entity it attaches to and the columns that + * reach it. + * + * WHICH TABLE those columns sit on depends on the relationship. On a foreign-key + * edge they are on the endpoint's own table. On a through-edge they are on the + * `through` table, referencing this endpoint entity's declared key -- the + * endpoints themselves hold nothing, which is the whole point of routing the + * edge through a separate table. */ export interface RelationshipEnd { entity: string; // name-reference into SemanticModel.entities - columns: string[]; // join columns on this endpoint's table + columns: string[]; // join columns; see above for which table they are on } /** - * An association (junction) table backing a many-to-many relationship. + * Why a many-to-many edge needs `Relationship.through`. * * A many-to-many link cannot be a foreign key: an FK column holds a single value * and so references at most one row (a to-one direction), which cannot encode a * pairing where each side maps to many of the other. The pairs instead live in a - * separate junction table, one row per (source, destination) -- e.g. an - * `enrollment` row per (student, course). + * table of their own, one row per (source, destination) -- an `enrollment` row + * per (student, course). That table is what `through` names. + * + * It stays a relationship rather than becoming its own construct: it has the + * same name, endpoints, and column mappings as any other edge, and both graph + * dialects render it as one more EDGE TABLE. What it adds is a table of its own, + * hence a key of its own (`keys`) and properties of its own (`fields`), and the + * endpoints' `columns` sitting on that table instead of on the endpoints. * - * Unlike a direct foreign key -- which the open format expresses and the loader - * produces -- a junction edge is backed by its OWN table (`dataSource`) with its - * OWN key (`keys`) and may carry edge `fields` (properties of the association - * itself, e.g. an enrollment's grade). Each side names the columns ON THE - * JUNCTION TABLE that reference the corresponding endpoint entity's declared - * `keys`. Authored as the `association` block on a relationship, which is a - * native key of the extended profile ('0.2.0.dev0/google') only -- vanilla - * Ossie has no junction-table syntax. See loader.associationSchema. + * `through`, `keys`, and `fields` are native keys of the extended profile + * ('0.2.0.dev0/google') only -- vanilla Ossie has no syntax for a table of + * pairs. See + * loader.refineRelationship for the rules that keep the two shapes from mixing. */ -export interface Association { - dataSource: string; // the junction table backing the edge - keys: string[]; // the edge's own key on the junction table - sourceColumns: string[]; // junction columns referencing the source entity's key - destinationColumns: string[]; // junction columns referencing the destination entity's key - fields?: Field[]; // edge properties (junction non-key columns) -} /** * A metric: a model-level, named aggregate. diff --git a/toolbox/mdcode/src/libts/semantic/kc_converter.ts b/toolbox/mdcode/src/libts/semantic/kc_converter.ts index c32e9d56..4048c809 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_converter.ts @@ -33,10 +33,10 @@ // (re-derived from its expression, as the loader does, when the catalog holds // one, else the value the emitter persisted), the model's deployment targets // (from the semantic-model aspect, back into the GOOGLE `custom_extensions` -// block), 1:1 / 1:N relationships (from the `schema-join` entry links a pull -// fetched -- see `modelsFromCatalogResources`'s `entryLinks` argument), and -// many-to-many relationships (from the `semantic-association` entries, which -// carry the junction table and both column pairs -- see kc_associations.ts). +// block), and relationships (from the `semantic-relationship` entries, which +// carry the whole edge -- see kc_relationships.ts -- falling back to the +// `schema-join` entry links a pull fetched, see `modelsFromCatalogResources`'s +// `entryLinks` argument, for a catalog written before those entries existed). // The per-field `semantics` block -- field/metric expressions and the DIMENSION // role // -- is gated off the catalog by default: the emitter writes it only under @@ -59,7 +59,7 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {Action, AiContext, CustomExtension, DataType, Entity, Field, Metric, Relationship, SemanticModel} from './ir'; import {isActionEntry, readAction} from './kc_actions'; -import {isAssociationEntry, readAssociation} from './kc_associations'; +import {isRelationshipEntry, readRelationshipEntry} from './kc_relationships'; import {referencedEntityNames} from './sql_expr_utils'; export interface ReadResult { @@ -93,7 +93,7 @@ export function modelsFromCatalogResources( // and recognizes an entry carrying it. A many-to-many relationship is the // same arrangement, in `kc_associations.ts`. const actionEntries = entries.filter(isActionEntry); - const associationEntries = entries.filter(isAssociationEntry); + const relationshipEntries = entries.filter(isRelationshipEntry); if (!anchors.length) { warnings.push('no semantic-model entry found; nothing to reconstruct'); @@ -128,16 +128,23 @@ export function modelsFromCatalogResources( entityEntriesForModel.forEach( (e, i) => entityByEntryId.set(idOf(e.name), entities[i])); - // A direct foreign-key relationship comes from the schema-join entry link - // whose two endpoints are both this model's entity entries. A many-to-many - // one is not a link but an entry of its own, so it is read separately and - // appended; the two together are the model's edges. - const relationships = [ - ...readRelationships(entryLinks, name, entityByEntryId, warnings), - ...childrenOf(anchor.name, associationEntries) - .map(e => readAssociation(e, entityNames, warnings)) - .filter((r): r is Relationship => r !== undefined), - ]; + // Relationships come from the `semantic-relationship` entries parented to + // this model, which hold the whole edge: its name verbatim, its endpoints + // and their columns, its instructions, and -- for a many-to-many edge -- + // the table it runs through and the edge's own properties. + // + // A push that wrote those entries wrote one for EVERY relationship, so when + // any are present they are the complete set, and the schema-join links are + // the same edges in lossier form -- reading both would double every + // foreign-key relationship. The links are the fallback instead, for a + // catalog last written by a kcmd that published links only. + const relationshipEntriesForModel = + childrenOf(anchor.name, relationshipEntries); + const relationships = relationshipEntriesForModel.length ? + relationshipEntriesForModel + .map(e => readRelationshipEntry(e, entityNames, warnings)) + .filter((r): r is Relationship => r !== undefined) : + readRelationships(entryLinks, name, entityByEntryId, warnings); const model: SemanticModel = {name, entities, relationships, metrics}; const description = anchor.entrySource?.description; @@ -159,7 +166,7 @@ export function modelsFromCatalogResources( // multiple anchors, where the sole-anchor fallback does not apply). if (!soleAnchor) { for (const child of [...entityEntries, ...metricEntries, ...actionEntries, - ...associationEntries]) { + ...relationshipEntries]) { if (!child.parentEntry || !anchorNames.has(child.parentEntry)) { warnings.push(`entry '${ child.name}' has no resolvable parent semantic-model; omitted`); diff --git a/toolbox/mdcode/src/libts/semantic/kc_custom_types.ts b/toolbox/mdcode/src/libts/semantic/kc_custom_types.ts index decbcf83..a1fbcaf0 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_custom_types.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_custom_types.ts @@ -21,8 +21,8 @@ // TO ADD A NEW CUSTOM TYPE. Append a record. Provisioning, naming and the init // wiring are generic over this list, so no other file in this directory needs // to change; what does need writing is the encoding that fills the aspect, the -// way kc_actions.ts does for `semantic-action` and kc_associations.ts does for -// `semantic-association`. +// way kc_actions.ts does for `semantic-action` and kc_relationships.ts does for +// `semantic-relationship`. // // A CUSTOM TYPE IS ONE ENTRY TYPE PLUS ONE ASPECT TYPE that share an id, the // way the built-in `semantic-metric` entry type and aspect type share theirs. @@ -197,19 +197,22 @@ const ACTION_ASPECT_TYPE: Omit = { }, }; -// The id of the association type. An association is the junction table backing -// a many-to-many relationship; kc_associations.ts holds the encoding that fills -// its aspect. -export const ASSOCIATION_TYPE_ID = 'semantic-association'; +// The id of the relationship type. kc_relationships.ts holds the encoding that +// fills its aspect. +export const RELATIONSHIP_TYPE_ID = 'semantic-relationship'; -// The aspect that carries a many-to-many relationship. +// The aspect that carries a relationship -- any relationship, one-to-many and +// many-to-many alike. // -// The built-in `schema-join` entry link already models a relationship, but only -// a direct foreign key: it holds ONE source/target column pair. A many-to-many -// edge is two joins through a third table, so it does not fit, and a custom -// entry LINK type is not available -- Dataplex accepts only its own link types. -// That leaves an entry, which is what this type is: one per many-to-many -// relationship, parented to the model anchor beside the entities and metrics. +// The built-in `schema-join` entry link already models a join, but only a direct +// foreign key: it holds ONE source/target column pair. A many-to-many edge is +// two joins through a third table, so it does not fit, and a custom entry LINK +// type is not available -- Dataplex accepts only its own link types. schema-join +// also has no field for the relationship's own name, which is why a link's name +// survives only in its lowercased, hyphenated id. That leaves an entry, which is +// what this type is: one per relationship, parented to the model anchor beside +// the entities and metrics. A foreign-key edge still emits its schema-join link +// as well, for the Dataplex surfaces that already read links. // // The endpoints are recorded as entity NAMES rather than as entry references // because the two entity entries are already the link endpoints a consumer @@ -217,19 +220,21 @@ export const ASSOCIATION_TYPE_ID = 'semantic-association'; // action parameter's `type`), and a name survives the project-number // normalization that rewrites resource names on the way back. // -// `fields` is the association's own properties -- an enrollment's grade, which -// belongs to neither endpoint. They ride here rather than in the built-in -// `schema` aspect for the same reason `instructions` does not use `guidelines`: -// a pull derives which aspect types to hydrate from the project the ENTRY type -// lives in, and a custom entry type lives in the destination project, where no -// built-in aspect type exists. -const ASSOCIATION_ASPECT_TYPE: Omit = { - displayName: 'Semantic Association', +// `through`, `keys` and `fields` are set only on a many-to-many edge: it is the +// one shape with a table of its own, hence a key of its own and properties -- +// an enrollment's grade -- belonging to neither endpoint. They ride here rather +// than in the built-in `schema` aspect for the same reason `instructions` does +// not use `guidelines`: a pull derives which aspect types to hydrate from the +// project the ENTRY type lives in, and a custom entry type lives in the +// destination project, where no built-in aspect type exists. +const RELATIONSHIP_ASPECT_TYPE: Omit = { + displayName: 'Semantic Relationship', description: - 'A many-to-many relationship in a semantic model: the two entities it ' + - 'pairs, and the junction table that holds the pairs.', + 'A relationship in a semantic model: the two entities it connects, the ' + + 'columns that reach them, and -- when it is many-to-many -- the table ' + + 'it runs through.', metadataTemplate: { - name: ASSOCIATION_TYPE_ID, + name: RELATIONSHIP_TYPE_ID, type: 'record', recordFields: [ { @@ -258,13 +263,15 @@ const ASSOCIATION_ASPECT_TYPE: Omit = { }, { index: 3, - name: 'junction', + name: 'through', type: 'string', annotations: { - displayName: 'Junction Table', + displayName: 'Through Table', description: - 'Resource name of the table holding the pairs, one row per ' + - '(from, to). Empty on a model with no physical binding.', + 'Resource name of the table a many-to-many edge runs through, ' + + 'holding one row per (from, to) pair. Empty on a foreign-key ' + + 'edge, which has no table of its own, and on a model with no ' + + 'physical binding.', }, }, { @@ -274,7 +281,10 @@ const ASSOCIATION_ASPECT_TYPE: Omit = { arrayItems: {name: 'key', type: 'string'}, annotations: { displayName: 'Keys', - description: 'The edge\'s own key columns on the junction table.', + description: + 'The edge\'s own key columns, on the table it runs through. ' + + 'Empty on a foreign-key edge, which is keyed by its source ' + + 'entity.', }, }, { @@ -285,7 +295,9 @@ const ASSOCIATION_ASPECT_TYPE: Omit = { annotations: { displayName: 'From Columns', description: - 'Junction-table columns referencing the from entity\'s key.', + 'Columns reaching the from entity: on that entity\'s own table ' + + 'for a foreign-key edge, on the through table for a ' + + 'many-to-many one.', }, }, { @@ -296,7 +308,9 @@ const ASSOCIATION_ASPECT_TYPE: Omit = { annotations: { displayName: 'To Columns', description: - 'Junction-table columns referencing the to entity\'s key.', + 'Columns reaching the to entity: on that entity\'s own table ' + + 'for a foreign-key edge, on the through table for a ' + + 'many-to-many one.', }, }, { @@ -336,7 +350,7 @@ const ASSOCIATION_ASPECT_TYPE: Omit = { annotations: { displayName: 'Expression', description: - 'The junction-table column the field binds to. Empty on ' + + 'The through-table column the field binds to. Empty on ' + 'a model with no physical binding.', }, }, @@ -345,8 +359,8 @@ const ASSOCIATION_ASPECT_TYPE: Omit = { annotations: { displayName: 'Fields', description: - 'Properties of the pairing itself, belonging to neither ' + - 'endpoint (an enrollment\'s grade).', + 'Properties of a many-to-many edge itself, belonging to ' + + 'neither endpoint (an enrollment\'s grade).', }, }, { @@ -375,14 +389,12 @@ export const CUSTOM_TYPES: readonly CustomType[] = [ aspectType: ACTION_ASPECT_TYPE, }, { - id: ASSOCIATION_TYPE_ID, + id: RELATIONSHIP_TYPE_ID, entryType: { - displayName: 'Semantic Association', - description: - 'A many-to-many relationship in a semantic model, backed by a ' + - 'junction table.', + displayName: 'Semantic Relationship', + description: 'A relationship between two entities of a semantic model.', }, - aspectType: ASSOCIATION_ASPECT_TYPE, + aspectType: RELATIONSHIP_ASPECT_TYPE, }, ]; diff --git a/toolbox/mdcode/src/libts/semantic/kc_associations.ts b/toolbox/mdcode/src/libts/semantic/kc_relationships.ts similarity index 50% rename from toolbox/mdcode/src/libts/semantic/kc_associations.ts rename to toolbox/mdcode/src/libts/semantic/kc_relationships.ts index f5172ea3..894a24b5 100644 --- a/toolbox/mdcode/src/libts/semantic/kc_associations.ts +++ b/toolbox/mdcode/src/libts/semantic/kc_relationships.ts @@ -1,38 +1,49 @@ -// How a model's MANY-TO-MANY relationships are encoded in Knowledge Catalog. +// How a model's relationships are encoded in Knowledge Catalog. // -// A one-to-many relationship publishes as a built-in `schema-join` entry link -// between the two entity entries. A many-to-many one cannot: schema-join holds -// a single source/target column pair, and a junction is two joins through a -// third table. A custom entry LINK type is not an option either -- Dataplex -// accepts only its own link types. So a many-to-many edge publishes as an ENTRY -// instead, one per relationship, parented to the model anchor beside the -// entities and metrics, carrying an aspect that holds both joins and the -// junction table. The type is the custom `semantic-association` pair DECLARED -// IN kc_custom_types.ts and created there by `kcmd init --semantic-model`. That -// file is the list of what is custom; this one is only the encoding that fills -// the aspect. +// Every relationship publishes as one ENTRY, parented to the model anchor +// beside the entities and metrics, carrying an aspect with the whole +// relationship: its name, its two endpoint entities, the columns that reach +// each of them, and -- for a many-to-many edge -- the table it runs through, +// that table's key, and the edge's own properties. The type is the custom +// `semantic-relationship` pair DECLARED IN kc_custom_types.ts and created there +// by `kcmd init +// --semantic-model`. That file is the list of what is custom; this one is only +// the encoding that fills the aspect. // -// The custom pair is what makes a many-to-many edge explicit in the catalog. A +// WHY AN ENTRY, when Dataplex has a built-in `schema-join` entry link for +// joins. schema-join holds exactly one source/target column pair, which cannot +// describe a many-to-many edge -- that is two joins through a third table -- +// and Dataplex has no custom entry LINK types to define one with. It also has +// no field for the relationship's own name, so a link's name survives only in +// its id, which is lowercased and hyphenated. An entry has room for all of it. +// +// A foreign-key relationship still ALSO emits its schema-join link (see +// knowledge_catalog.relationshipLink). The entry is the fidelity record, the +// link is the graph-shaped projection that other Dataplex surfaces already +// understand; dropping the link would trade interop for nothing. The two agree, +// and a pull prefers the entry (see kc_converter). +// +// The custom pair is what makes a relationship explicit in the catalog. A // search can tell one apart from anything else by its entry type, and the // aspect's fields are typed and queryable rather than prose a reader has to // interpret. // -// WHEN A BUILT-IN MANY-TO-MANY TYPE SHIPS, follow the instructions at the top +// WHEN A BUILT-IN RELATIONSHIP TYPE SHIPS, follow the instructions at the top // of kc_custom_types.ts. Nothing in this file changes: the readers below match -// a type by its id suffix, so they do not care which project it lives in. If -// what ships instead is a schema-join that can model a junction, this file goes -// away and `relationshipLink` in knowledge_catalog.ts stops skipping M:N. +// a type by its id suffix, so they do not care which project it lives in. // // The call sites are `knowledge_catalog.ts` (emit the entries), // `kc_converter.ts` (read them back), and `pull_kc.ts` (hydrate the aspect). // -// `instructions` and the edge's own `fields` ride the association's aspect +// `instructions` and the edge's own `fields` ride the relationship's aspect // rather than the built-in `guidelines` and `schema` aspects an entity uses. A // pull derives which aspect types to hydrate from the project the ENTRY type -// lives in, so an association entry, whose type is custom and therefore in the +// lives in, so a relationship entry, whose type is custom and therefore in the // destination project, would ask for aspect types that exist only under -// `dataplex-types`. Keeping both on the association's own aspect keeps the -// whole encoding inside the one type kc_custom_types.ts provisions. +// `dataplex-types`. Keeping both on the relationship's own aspect keeps the +// whole encoding inside the one type kc_custom_types.ts provisions. It is also +// the only home either has ever had: the `guidelines` aspect attaches to a +// model, an entity, or a metric, never to a relationship. // // The helpers at the bottom duplicate a few lines from the modules above on // purpose. This module imports only the IR, the entry shape and the type @@ -40,39 +51,39 @@ import {Entry} from '../gcp/dataplex'; -import {AiContext, Association, DATA_TYPES, DataType, Field, Relationship, SemanticModel} from './ir'; -import {ASSOCIATION_TYPE_ID, customAspectKey, customAspectTypeName, customEntryTypeName} from './kc_custom_types'; +import {AiContext, DATA_TYPES, DataType, Field, Relationship, SemanticModel} from './ir'; +import {customAspectKey, customAspectTypeName, customEntryTypeName, RELATIONSHIP_TYPE_ID} from './kc_custom_types'; -// Full resource name of the association entry type for a destination. -export function associationEntryTypeName(dest: {project: string}): string { - return customEntryTypeName(ASSOCIATION_TYPE_ID, dest); +// Full resource name of the relationship entry type for a destination. +export function relationshipEntryTypeName(dest: {project: string}): string { + return customEntryTypeName(RELATIONSHIP_TYPE_ID, dest); } -// Full resource name of the association aspect type for a destination. -export function associationAspectTypeName(dest: {project: string}): string { - return customAspectTypeName(ASSOCIATION_TYPE_ID, dest); +// Full resource name of the relationship aspect type for a destination. +export function relationshipAspectTypeName(dest: {project: string}): string { + return customAspectTypeName(RELATIONSHIP_TYPE_ID, dest); } // Aspect-map key: the `project.location.type` reference form the client keys an // entry's aspects by. -export function associationAspectKey(dest: {project: string}): string { - return customAspectKey(ASSOCIATION_TYPE_ID, dest); +export function relationshipAspectKey(dest: {project: string}): string { + return customAspectKey(RELATIONSHIP_TYPE_ID, dest); } // --------------------------------------------------------------------------- -// Write side: the IR -> one entry per many-to-many relationship. +// Write side: the IR -> one entry per relationship. // --------------------------------------------------------------------------- // What the emitter supplies so this file need not rebuild entry names or repeat // the id-collision bookkeeping it already does for entities and metrics. -export interface AssociationEmitContext { +export interface RelationshipEmitContext { // The destination project, which is where the custom types live. project: string; // Full entry resource name for an entry id (Namer.entry). entry(entryId: string): string; // Full entry resource name of the model anchor, the parent of every - // association. + // relationship. anchor: string; // Reserves an entry id, returning false when it collides with one already // emitted (knowledge_catalog.claim). @@ -82,39 +93,39 @@ export interface AssociationEmitContext { // ending on it would name an entity the catalog has no entry for. publishedEntities: Set; // Renders a table into the linked-resource form the catalog stores - // (knowledge_catalog.resourcePath), so the junction is addressed the same way - // an entity's backing table is. + // (knowledge_catalog.resourcePath), so the table a many-to-many edge runs + // through is addressed the same way an entity's backing table is. resource(dataSource: string): string; } -// The entry id of one association: `.associations.`, alongside +// The entry id of one relationship: `.relationships.`, alongside // `.entities.` and `.metrics.`. -export function associationEntryId(modelId: string, relName: string): string { - return `${modelId}.associations.${slug(relName)}`; +export function relationshipEntryId(modelId: string, relName: string): string { + return `${modelId}.relationships.${slug(relName)}`; } -// The entry-id prefix a model's associations occupy, so delete reconciliation +// The entry-id prefix a model's relationships occupy, so delete reconciliation // removes the entry of a relationship dropped from the model. -export function associationOwnedPrefix(modelId: string): string { - return `${modelId}.associations.`; +export function relationshipOwnedPrefix(modelId: string): string { + return `${modelId}.relationships.`; } /** - * One entry per many-to-many relationship, to append to the model's entries. + * One entry per relationship, to append to the model's entries. * - * Empty when the model declares none, so a model of only foreign-key edges is - * unchanged from before this type existed. An edge whose endpoint entity this - * push does not publish is skipped with a warning, the same way - * `relationshipLink` skips a foreign-key edge in that situation. + * Every relationship gets one, foreign-key and many-to-many alike -- including + * a purely logical edge with no join columns, which has no schema-join link to + * be published by and would otherwise reach the catalog not at all. + * + * An edge whose endpoint entity this push does not publish is skipped with a + * warning; `relationshipLink` then skips the same edge silently, so one dropped + * relationship reports once. */ -export function associationEntries( - model: SemanticModel, modelId: string, ctx: AssociationEmitContext, +export function relationshipEntries( + model: SemanticModel, modelId: string, ctx: RelationshipEmitContext, warnings: string[]): Entry[] { const entries: Entry[] = []; for (const rel of model.relationships ?? []) { - const assoc = rel.association; - if (!assoc) continue; - const missing = !ctx.publishedEntities.has(rel.source.entity) ? rel.source.entity : !ctx.publishedEntities.has(rel.destination.entity) ? @@ -123,24 +134,24 @@ export function associationEntries( if (missing !== undefined) { warnings.push( `relationship '${rel.name}': endpoint entity '${missing}' is not a ` + - `published entity; the many-to-many relationship is skipped.`); + `published entity; the relationship is skipped.`); continue; } - const id = associationEntryId(modelId, rel.name); - if (!ctx.claim(id, `many-to-many relationship '${rel.name}'`)) continue; + const id = relationshipEntryId(modelId, rel.name); + if (!ctx.claim(id, `relationship '${rel.name}'`)) continue; entries.push({ name: ctx.entry(id), - entryType: associationEntryTypeName(ctx), + entryType: relationshipEntryTypeName(ctx), parentEntry: ctx.anchor, entrySource: compact({ displayName: rel.name, description: rel.description, }) as Entry['entrySource'], aspects: { - [associationAspectKey(ctx)]: { - aspectType: associationAspectTypeName(ctx), - data: associationAspectData(rel, assoc, ctx), + [relationshipAspectKey(ctx)]: { + aspectType: relationshipAspectTypeName(ctx), + data: relationshipAspectData(rel, ctx), }, }, }); @@ -148,23 +159,24 @@ export function associationEntries( return entries; } -// The aspect payload for one many-to-many relationship: its two endpoints, the -// junction table and the columns on it that reach each endpoint, the edge's own -// properties, and any AI instructions. -function associationAspectData( - rel: Relationship, assoc: Association, - ctx: AssociationEmitContext): Record { +// The aspect payload for one relationship: its two endpoints and the columns +// that reach them, the table a many-to-many edge runs through with that table's +// key and the edge's properties, and any AI instructions. +function relationshipAspectData( + rel: Relationship, ctx: RelationshipEmitContext): Record { return compact({ fromEntity: rel.source.entity, toEntity: rel.destination.entity, - // A model with no physical binding has no junction table to name; the - // template leaves the field optional, so omit it rather than store ''. - junction: ctx.resource(assoc.dataSource) || undefined, - keys: nonEmpty(assoc.keys), - fromColumns: nonEmpty(assoc.sourceColumns), - toColumns: nonEmpty(assoc.destinationColumns), + // Set only on a many-to-many edge. A model with no physical binding has no + // table to name either; the template leaves the field optional, so omit it + // rather than store ''. + through: rel.through ? ctx.resource(rel.through) || undefined : undefined, + keys: nonEmpty(rel.keys ?? []), + // Absent on a purely logical edge, which is bound by nothing yet. + fromColumns: nonEmpty(rel.source.columns), + toColumns: nonEmpty(rel.destination.columns), fields: nonEmpty( - (assoc.fields ?? + (rel.fields ?? []).map(f => compact({ name: f.name, // An untyped field is published as Opaque, the explicit @@ -182,39 +194,43 @@ function associationAspectData( // --------------------------------------------------------------------------- -// Read side: an association entry -> the IR. +// Read side: a relationship entry -> the IR. // --------------------------------------------------------------------------- -// True when an entry is one of a model's many-to-many relationships, matched by -// the entry type's id suffix so the project the type lives in need not be known -// -- which is what lets a pull keep working when the custom type is replaced by -// a built-in one. -export function isAssociationEntry(entry: Entry): boolean { - return entry.entryType?.endsWith(`/entryTypes/${ASSOCIATION_TYPE_ID}`) ?? +// True when an entry is one of a model's relationships, matched by the entry +// type's id suffix so the project the type lives in need not be known -- which +// is what lets a pull keep working when the custom type is replaced by a +// built-in one. +export function isRelationshipEntry(entry: Entry): boolean { + return entry.entryType?.endsWith(`/entryTypes/${RELATIONSHIP_TYPE_ID}`) ?? false; } -// The aspect type resource names to hydrate for an association entry. Named +// The aspect type resource names to hydrate for a relationship entry. Named // through the entry type's own project so the pull follows the type wherever it // lives. -export function associationAspectTypes(entryTypeBase: string): string[] { - return [`${entryTypeBase}/aspectTypes/${ASSOCIATION_TYPE_ID}`]; +export function relationshipAspectTypes(entryTypeBase: string): string[] { + return [`${entryTypeBase}/aspectTypes/${RELATIONSHIP_TYPE_ID}`]; } /** - * Recovers one many-to-many relationship from its entry, the inverse of - * associationEntries. + * Recovers one relationship from its entry, the inverse of + * relationshipEntries. * * Returns undefined, with a warning, for an entry naming an endpoint that is - * not one of the model's entities, or missing a junction column list: an edge - * that cannot say what it joins is not a usable relationship, and one bad entry - * degrades itself rather than the pull. + * not one of the model's entities, or for a many-to-many edge whose column + * lists are empty: an edge running through a table but not saying which pairs + * it holds cannot be reloaded. One bad entry degrades itself rather than the + * pull. + * + * Empty column lists on an edge with no `through` are fine -- that is a purely + * logical relationship, and recovering it is the point of storing one. */ -export function readAssociation( +export function readRelationshipEntry( entry: Entry, entityNames: string[], warnings: string[]): Relationship| undefined { const name = entry.entrySource?.displayName || idOf(entry.name); - const data = associationAspectDataOf(entry); + const data = relationshipAspectDataOf(entry); const known = new Set(entityNames); const from = str(data.fromEntity); @@ -222,43 +238,37 @@ export function readAssociation( for (const [side, end] of [['fromEntity', from], ['toEntity', to]] as const) { if (!known.has(end)) { warnings.push( - `many-to-many relationship '${name}': ${side} '${end}' is not one ` + - `of the model's entities; the relationship is skipped`); + `relationship '${name}': ${side} '${end}' is not one of the ` + + `model's entities; the relationship is skipped`); return undefined; } } const fromColumns = stringList(data.fromColumns); const toColumns = stringList(data.toColumns); - if (!fromColumns.length || !toColumns.length) { + const through = dataSourceFromResource(str(data.through)); + if (through && (!fromColumns.length || !toColumns.length)) { const side = !fromColumns.length ? 'from' : 'to'; warnings.push( - `many-to-many relationship '${name}': the ${ASSOCIATION_TYPE_ID} ` + - `aspect names no junction column on the ${side} end; the ` + - `relationship is skipped`); + `relationship '${name}': the ${RELATIONSHIP_TYPE_ID} aspect names a ` + + `'through' table but no column on the ${side} end; the relationship ` + + `is skipped`); return undefined; } - const association: Association = { - dataSource: dataSourceFromResource(str(data.junction)), - keys: stringList(data.keys), - sourceColumns: fromColumns, - destinationColumns: toColumns, - }; - const fields = asArray(data.fields) - .map(f => readField(f)) - .filter((f): f is Field => f !== undefined); - if (fields.length) association.fields = fields; - - // A many-to-many edge carries no columns on either ENDPOINT table: the - // columns that bind it are the junction's, above. Empty lists here are what - // the loader produces for an authored `association`, so the two agree. const relationship: Relationship = { name, - source: {entity: from, columns: []}, - destination: {entity: to, columns: []}, - association, + source: {entity: from, columns: fromColumns}, + destination: {entity: to, columns: toColumns}, }; + if (through) { + relationship.through = through; + relationship.keys = stringList(data.keys); + const fields = asArray(data.fields) + .map(f => readField(f)) + .filter((f): f is Field => f !== undefined); + if (fields.length) relationship.fields = fields; + } const description = entry.entrySource?.description; if (description !== undefined && description !== '') { relationship.description = description; @@ -290,14 +300,14 @@ function readField(f: any): Field|undefined { // Local helpers (see the file header on why they are not shared). // --------------------------------------------------------------------------- -// The association aspect's `data` from an entry, matched by the aspect key's -// `.semantic-association` suffix or the aspectType's -// `/aspectTypes/semantic-association` suffix, so it is found whichever project +// The relationship aspect's `data` from an entry, matched by the aspect key's +// `.semantic-relationship` suffix or the aspectType's +// `/aspectTypes/semantic-relationship` suffix, so it is found whichever project // the type was provisioned in. -function associationAspectDataOf(entry: Entry): Record { +function relationshipAspectDataOf(entry: Entry): Record { for (const [key, aspect] of Object.entries(entry.aspects ?? {})) { - if (key.endsWith(`.${ASSOCIATION_TYPE_ID}`) || - aspect.aspectType?.endsWith(`/aspectTypes/${ASSOCIATION_TYPE_ID}`)) { + if (key.endsWith(`.${RELATIONSHIP_TYPE_ID}`) || + aspect.aspectType?.endsWith(`/aspectTypes/${RELATIONSHIP_TYPE_ID}`)) { return aspect.data ?? {}; } } diff --git a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts index 0b7ac1ae..15ba07ac 100644 --- a/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts +++ b/toolbox/mdcode/src/libts/semantic/knowledge_catalog.ts @@ -42,20 +42,26 @@ // system-type templates yet, so they are gated behind // KcGenerateOptions.emitExpressions (off by default) and omitted above. // -// A direct foreign-key relationship becomes a `schema-join` entry link between -// the two entity entries. schema-join is a built-in, undirected entry link type -// in `dataplex-types/global` whose required `schema-join` aspect carries the -// join detail (the paired join columns, JOIN vs FOREIGN_KEY, USER inference). -// The join's direction -- which side holds the foreign key -- is preserved -// inside that aspect, not by the link. +// EVERY relationship becomes an ENTRY parented to the anchor, using the same +// custom-type mechanism as an action: `kc_custom_types.ts` declares the +// `semantic-relationship` pair and `kc_relationships.ts` encodes the aspect. +// This module only appends the entries it returns. That entry is the fidelity +// record -- it has room for the relationship's own name, its instructions, and +// (on a many-to-many edge) the table it runs through and the edge's own +// properties, none of which a link can hold. // -// A MANY-TO-MANY relationship is not a link at all. A junction is two joins -// through a third table, which schema-join's single source/target pair does not -// model, and Dataplex has no custom entry LINK types to define one with. It -// publishes as an ENTRY instead, one per relationship parented to the anchor, -// using the same custom-type mechanism as an action: `kc_custom_types.ts` -// declares the `semantic-association` pair and `kc_associations.ts` encodes the -// aspect. This module only appends the entries it returns. +// A DIRECT FOREIGN-KEY relationship ALSO becomes a `schema-join` entry link +// between the two entity entries. schema-join is a built-in, undirected entry +// link type in `dataplex-types/global` whose required `schema-join` aspect +// carries the join detail (the paired join columns, JOIN vs FOREIGN_KEY, USER +// inference). The join's direction -- which side holds the foreign key -- is +// preserved inside that aspect, not by the link. It is kept alongside the entry +// because other Dataplex surfaces already read links; the two agree, and a pull +// prefers the entry. +// +// A relationship with a `through` table gets no link: an edge through a third +// table is two joins, which schema-join's single source/target pair does not +// model, and Dataplex has no custom entry LINK types to define one with. // import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; @@ -63,7 +69,7 @@ import type {Aspect, Entry, EntryLink} from '../gcp/dataplex'; import {googleDeploymentTargets} from './deployment_target'; import {AiContext, DataType, Entity, Metric, Relationship, SemanticModel} from './ir'; import {actionEntries, actionOwnedPrefix} from './kc_actions'; -import {associationEntries, associationOwnedPrefix} from './kc_associations'; +import {relationshipEntries, relationshipOwnedPrefix} from './kc_relationships'; // Where the `semantic-*` and `schema` system types live: built-in types in // project `dataplex-types`, location `global`. Callers may override to reference @@ -223,24 +229,23 @@ export function generateCatalogResources( publishedEntities: new Set(entityEntryName.keys()), }, warnings)); - // One entry per many-to-many relationship, for the same reason: a junction - // has no built-in type either, so `kc_associations.ts` fills the custom - // `semantic-association` aspect declared in `kc_custom_types.ts`. It returns - // nothing when every relationship is a direct foreign key. - entries.push(...associationEntries(model, modelId, { + // One entry per relationship, for the same reason: there is no built-in + // relationship ENTRY type either, so `kc_relationships.ts` fills the custom + // `semantic-relationship` aspect declared in `kc_custom_types.ts`. + entries.push(...relationshipEntries(model, modelId, { project: opts.project, entry: (id: string) => names.entry(id), anchor: modelEntryName, claim: (id: string, label: string) => claim(seen, id, 'entry', label, warnings), publishedEntities: new Set(entityEntryName.keys()), - // The junction table is addressed the same way an entity's backing table - // is, so the emitter's own mapping is what renders it. + // The table a many-to-many edge runs through is addressed the same way an + // entity's backing table is, so the emitter's own mapping renders it. resource: resourcePath, }, warnings)); - // A direct foreign-key relationship maps to a schema-join entry link between - // its endpoint entries. A many-to-many one became an entry above. + // A direct foreign-key relationship ALSO maps to a schema-join entry link + // between its endpoint entries, alongside the entry emitted above. const entryLinks: EntryLink[] = []; const seenLinks = new Set(); for (const rel of relationships) { @@ -254,50 +259,42 @@ export function generateCatalogResources( entryLinks, warnings: [...new Set(warnings)], // Ossie ids are dotted: `.entities.` / `.metrics.` - // / `.actions.` / `.associations.`. + // / `.actions.` / `.relationships.`. ownedPrefixes: [ `${modelId}.entities.`, `${modelId}.metrics.`, actionOwnedPrefix(modelId), - associationOwnedPrefix(modelId), + relationshipOwnedPrefix(modelId), ], }; } // Builds the schema-join entry link for one direct foreign-key relationship, or -// undefined when it cannot be published. A many-to-many edge returns undefined -// without a warning: it is not a link, and `associationEntries` has already -// published it as an entry. Two cases are skipped WITH a warning: an edge whose -// endpoint entity was not emitted (e.g. skipped for a duplicate id), and a -// column-less (purely logical) edge, whose join columns must be added to the -// model before it can publish. The BigQuery property graph still carries the -// latter once bound. +// undefined when there is no link to build. +// +// Every skip here is SILENT, because the link is no longer the relationship's +// only route into the catalog -- `relationshipEntries` has already published +// the relationship itself, so a missing link is a missing projection, not lost +// metadata. Three cases skip: a many-to-many edge (not a link at all), an edge +// whose endpoint entity was not emitted (the entry emitter warned about that +// one already, and warning twice for one relationship reads as two problems), +// and a column-less purely logical edge. function relationshipLink( names: Namer, model: SemanticModel, rel: Relationship, entityEntryName: Map, seenLinks: Set, warnings: string[]): EntryLink|undefined { - if (rel.association) return undefined; + if (rel.through) return undefined; const src = entityEntryName.get(rel.source.entity); const dst = entityEntryName.get(rel.destination.entity); - if (!src || !dst) { - const missing = !src ? rel.source.entity : rel.destination.entity; - warnings.push( - `relationship '${rel.name}': endpoint entity '${missing}' is not a ` + - `published entity; the relationship link is skipped.`); - return undefined; - } + if (!src || !dst) return undefined; // A purely logical edge (an OWL import) carries no join columns. schema-join // is a server-defined CLOSED metadataTemplate whose `fields` requirement we // cannot depend on, so rather than risk a rejected aspect on a KC push we skip - // the link until the edge is bound. Add the relationship's from_columns / - // to_columns to the model and it publishes; the edge still lives in the model - // (and, once bound, the BigQuery/Spanner graph). + // the link until the edge is bound. The relationship itself is published + // either way, by its own entry; add its from_columns / to_columns to the + // model and the link appears too. if (!rel.source.columns.length || !rel.destination.columns.length) { - warnings.push( - `relationship '${rel.name}': no join columns, so it is not published ` + - `to Knowledge Catalog yet; add its from_columns and to_columns to the ` + - `relationship in the model to publish the link.`); return undefined; } const linkId = names.linkId(model, rel); @@ -306,18 +303,11 @@ function relationshipLink( warnings)) return undefined; - // The name lives only in the link id (schema-join's aspect has no name - // field), and link ids are normalized -- lowercase, hyphens only. When the - // authored name is not already in that form, a later pull recovers it - // lowercased and hyphenated, not verbatim; warn so the round-trip change is - // not a surprise. - const normalizedName = linkSlug(rel.name); - if (normalizedName !== rel.name) { - warnings.push( - `relationship '${rel.name}': Knowledge Catalog stores the name only ` + - `in the normalized link id, so a pull returns it lowercased/hyphenated ` + - `(e.g. '${normalizedName}'), not '${rel.name}'.`); - } + // The link's own name survives only in its id, which is normalized to + // lowercase and hyphens (schema-join's aspect has no name field). That used + // to be how a pull recovered the name, and so used to cost the authored + // casing; the relationship entry now carries the name verbatim and a pull + // reads it from there, so the normalized id is just an id. return { name: names.entryLink(linkId), diff --git a/toolbox/mdcode/src/libts/semantic/loader.ts b/toolbox/mdcode/src/libts/semantic/loader.ts index 5400d3d5..fb3a8db1 100644 --- a/toolbox/mdcode/src/libts/semantic/loader.ts +++ b/toolbox/mdcode/src/libts/semantic/loader.ts @@ -12,7 +12,7 @@ import * as yaml from 'yaml'; import * as z from 'zod'; -import {Action, ActionParameter, AiContext, Association, CustomExtension, DATA_TYPES, Entity, Executor, Field, Metric, Relationship, SemanticModel,} from './ir'; +import {Action, ActionParameter, AiContext, CustomExtension, DATA_TYPES, Entity, Executor, Field, Metric, Relationship, SemanticModel,} from './ir'; import {referencedEntityNames} from './sql_expr_utils'; export interface LoadOptions { @@ -154,24 +154,13 @@ const datasetBase = z.object({ // the base shape above and the strict per-load schemas in buildDocumentSchema // so the two cannot drift. // -// A direct foreign key binds the edge with the endpoints' own columns; a -// junction table binds it with the junction's. They are alternatives, so a -// relationship carries one set or the other, never both and never half of one. +// Both shapes of edge bind with `from_columns`/`to_columns`; `through` changes +// which table those columns are on, and brings a key and properties of its own +// that a foreign-key edge has nowhere to put. function refineRelationship( r: {name: string; from_columns?: string[]; to_columns?: string[]; - association?: unknown}, + through?: unknown; keys?: unknown; fields?: unknown}, ctx: z.RefinementCtx) { - if (r.association !== undefined && - (r.from_columns !== undefined || r.to_columns !== undefined)) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: `relationship '${r.name}': a many-to-many edge is bound by its ` + - `association's from_columns/to_columns (columns on the junction ` + - `table), so the relationship's own from_columns/to_columns must be ` + - `removed -- neither endpoint holds a foreign key.`, - }); - return; - } if ((r.from_columns === undefined) !== (r.to_columns === undefined)) { ctx.addIssue({ code: z.ZodIssueCode.custom, @@ -181,40 +170,47 @@ function refineRelationship( `without the other is a half-bound join.`, }); } + if (r.through !== undefined && r.from_columns === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `relationship '${r.name}': an edge through '${ + r.through}' must give from_columns and to_columns -- the ` + + `columns on that table reaching each endpoint. Without them nothing ` + + `says which pairs it holds.`, + }); + } + for (const [key, value] of + [['keys', r.keys], ['fields', r.fields]] as const) { + if (value !== undefined && r.through === undefined) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `relationship '${r.name}': '${key}' needs a 'through' table. ` + + `A foreign-key edge is carried by its source entity's table, so it ` + + `has no table of its own to hold ${ + key === 'keys' ? 'a key' : 'properties'}.`, + }); + } + } } -// The junction table backing a MANY-TO-MANY relationship (GOOGLE_VERSION -// only; see Association in ./ir). -// -// A many-to-many link cannot be a foreign key: an FK column holds one value and -// so references at most one row. The pairs live in a table of their own -// instead, one row per (from, to) -- an `enrollment` row per (student, course). -// That table is what `source` names. -// -// The `from_columns`/`to_columns` HERE are columns on the JUNCTION table, each -// referencing the corresponding endpoint entity's declared key. That is why the -// relationship's own `from_columns`/`to_columns` must be absent when this block -// is present: neither endpoint holds a foreign key. -const associationSchema = z.object({ - source: z.string(), - // The edge's own key on the junction table. Optional: when omitted the - // generators key the edge by its two endpoint column lists, deduplicated. - keys: z.array(z.string()).min(1).optional(), - from_columns: z.array(z.string()).min(1), - to_columns: z.array(z.string()).min(1), - // Properties of the pairing itself -- an enrollment's grade. Same shape as an - // entity's fields. - fields: z.array(fieldBase).optional(), -}); - const relationshipSchema = z.object({ name: z.string(), from: z.string(), to: z.string(), - // Present only on a many-to-many edge, which is - // backed by a junction table rather than by a - // foreign key (GOOGLE_VERSION only). - association: associationSchema.optional(), + // The table the edge runs THROUGH, present only on + // a many-to-many edge (GOOGLE_VERSION only). It + // holds one row per pair, so it is what + // `from_columns`/`to_columns` are columns ON. See + // Relationship.through in ./ir. + through: z.string().optional(), + // The edge's own key on the `through` table. + // Optional: when omitted the generators key the + // edge by its two column lists, deduplicated. + keys: z.array(z.string()).min(1).optional(), + // Properties of the pairing itself -- an + // enrollment's grade. Same shape as an entity's + // fields. + fields: z.array(fieldBase).optional(), // Join columns are the physical binding of the // edge and are OPTIONAL, so a purely logical // relationship (an ontology edge, direction only) @@ -401,26 +397,18 @@ function buildDocumentSchema(bindingOptional: boolean, extended: boolean) { // either `bindingOptional`. }); - // A field inside an association is the same shape as an entity's, minus the + // An edge property is the same shape as an entity's field, minus the // extension carrier the version does not allow. - const associationField = z.object({ - name: z.string(), - expression: expressionSchema.optional(), - datatype: z.enum(DATA_TYPES).optional(), - description: z.string().optional(), - label: z.string().optional(), - dimension: dimensionSchema.optional(), - ai_context: aiContextSchema.optional(), - ...ce, - }).strict(); - - const association = z.object({ - source: z.string(), - keys: z.array(z.string()).min(1).optional(), - from_columns: z.array(z.string()).min(1), - to_columns: z.array(z.string()).min(1), - fields: z.array(associationField).optional(), - }).strict(); + const edgeField = z.object({ + name: z.string(), + expression: expressionSchema.optional(), + datatype: z.enum(DATA_TYPES).optional(), + description: z.string().optional(), + label: z.string().optional(), + dimension: dimensionSchema.optional(), + ai_context: aiContextSchema.optional(), + ...ce, + }).strict(); const relationship = z.object({ @@ -433,10 +421,15 @@ function buildDocumentSchema(bindingOptional: boolean, extended: boolean) { ai_context: aiContextSchema.optional(), ...ce, // Many-to-many is a native extension: vanilla Ossie has no - // junction-table syntax and no `custom_extensions` encoding for one, - // so under OSSIE_VERSION the key is rejected as unknown rather than - // silently dropped. - ...(extended ? {association: association.optional()} : {}), + // syntax for a table of pairs and no `custom_extensions` encoding, + // so under OSSIE_VERSION these keys are rejected as unknown rather + // than silently dropped. + ...(extended ? { + through: z.string().optional(), + keys: z.array(z.string()).min(1).optional(), + fields: z.array(edgeField).optional(), + } : + {}), }) .strict() .superRefine((r, ctx) => { @@ -518,7 +511,6 @@ type ExpressionDoc = z.infer; type DatasetDoc = z.infer; type FieldDoc = z.infer; type RelationshipDoc = z.infer; -type AssociationDoc = z.infer; type MetricDoc = z.infer; type ModelDoc = z.infer; type ActionDoc = z.infer; @@ -847,11 +839,13 @@ function convertField( return field; } -// Maps an OSI foreign-key relationship onto the IR edge. `source.columns` are -// the FK columns on the `from` table (`from_columns`); `destination.columns` -// are the referenced key columns on the `to` table (`to_columns`), paired -// positionally. A logical relationship carries no columns (both endpoints -// empty); a graph push requires them and rejects a column-less edge (see +// Maps an OSI relationship onto the IR edge. `source.columns` and +// `destination.columns` are `from_columns` and `to_columns`; which table those +// columns sit on depends on the edge. A foreign-key edge puts `from_columns` on +// the `from` table and `to_columns` on the `to` table, paired positionally; an +// edge with a `through` table puts both on that table, one list reaching each +// endpoint. A logical relationship carries no columns (both endpoints empty); a +// graph push requires them and rejects a column-less edge (see // validatePushRequirements). The source entity's own primary key is not // duplicated here -- downstream consumers look it up from the entity. A // malformed relationship (an endpoint not declared in the model, or mismatched @@ -871,7 +865,11 @@ function convertRelationship( } const fromColumns = r.from_columns ?? []; const toColumns = r.to_columns ?? []; - if (fromColumns.length !== toColumns.length) { + // A foreign key pairs its two lists positionally, so their lengths must + // match. A through table's do not pair with each other at all -- each list + // reaches a different entity's key -- so a composite key on one side and a + // single column on the other is valid there. + if (r.through === undefined && fromColumns.length !== toColumns.length) { throw new Error( `${ctx}: from_columns (${fromColumns.length}) and to_columns ` + `(${ @@ -884,9 +882,8 @@ function convertRelationship( source: {entity: r.from, columns: fromColumns}, destination: {entity: r.to, columns: toColumns}, }; - if (r.association) { - relationship.association = - convertAssociation(r.association, r.name, opts, warnings, dialect); + if (r.through !== undefined) { + bindThrough(relationship, r, opts, warnings, dialect); } const description = composeDescription(r.description); if (description) relationship.description = description; @@ -897,32 +894,33 @@ function convertRelationship( return relationship; } -// Maps the junction-table block of a many-to-many relationship onto the IR's -// Association. +// Fills in the through-table half of a many-to-many relationship. The endpoint +// columns are already on `relationship` -- they are the same authored keys a +// foreign-key edge uses, just pointing at the through table -- so this adds only +// what a through-edge has extra: the table, its key, and its properties. // // `keys` is the edge's own key. The format leaves it optional because the pair // of endpoint column lists is already unique in the common case, so when it is // omitted the key is those two lists concatenated and deduplicated -- the same // default the BigQuery and Spanner generators would otherwise have to invent -// separately. Give it explicitly when the junction has a surrogate key, or when -// a pair may legitimately repeat (an enrollment per term). -function convertAssociation( - a: AssociationDoc, relName: string, opts: LoadOptions, warnings: string[], - dialect: string): Association { - const ctxLabel = `relationship '${relName}' association`; - const fields = - (a.fields ?? []).map(f => convertField(f, relName, warnings, dialect)); +// separately. Give it explicitly when the table has a surrogate key, or when a +// pair may legitimately repeat (an enrollment per term). +function bindThrough( + relationship: Relationship, + r: {name: string; through?: string; keys?: string[]; fields?: FieldDoc[]}, + opts: LoadOptions, warnings: string[], dialect: string) { + const ctxLabel = `relationship '${relationship.name}'`; + const fields = (r.fields ?? []) + .map(f => convertField(f, relationship.name, warnings, + dialect)); rejectDuplicateNames(fields.map(f => f.name), 'field name', ctxLabel); - const keys = a.keys ?? [...new Set([...a.from_columns, ...a.to_columns])]; - const association: Association = { - dataSource: parseSource(a.source, opts, warnings, ctxLabel), - keys, - sourceColumns: a.from_columns, - destinationColumns: a.to_columns, - }; - if (fields.length) association.fields = fields; - return association; + relationship.through = parseSource(r.through!, opts, warnings, ctxLabel); + relationship.keys = r.keys ?? + [...new Set([ + ...relationship.source.columns, ...relationship.destination.columns + ])]; + if (fields.length) relationship.fields = fields; } function convertMetric( diff --git a/toolbox/mdcode/src/libts/semantic/osi_converter.ts b/toolbox/mdcode/src/libts/semantic/osi_converter.ts index ab351a62..96423c29 100644 --- a/toolbox/mdcode/src/libts/semantic/osi_converter.ts +++ b/toolbox/mdcode/src/libts/semantic/osi_converter.ts @@ -36,13 +36,14 @@ // any other vendor extension on the IR is dropped with a warning (its carrier's // fate under '/google' is still open). See serialize.test.ts. // -// A many-to-many relationship round-trips whole: its `association` block is a -// native key of the extended profile, so the junction table, the edge's key, -// the junction-side columns and the edge properties are all written back. +// A many-to-many relationship round-trips whole: `through`, `keys`, and +// `fields` are native keys of the extended profile, so the table the edge runs +// through, its own key, the columns on it, and its properties are all written +// back. import * as yaml from 'yaml'; -import {Action, AiContext, Association, CustomExtension, Entity, Executor, Field, Metric, Relationship, SemanticModel,} from './ir'; +import {Action, AiContext, CustomExtension, Entity, Executor, Field, Metric, Relationship, SemanticModel,} from './ir'; // The version stamped on every serialized document. Pull emits kcmd's extended // profile: it uses native extension keys (`entities`, `deployment_target`) @@ -81,8 +82,8 @@ export interface SerializeResult { * `pull` writes one file per model * (catalog/EntryGroups//.yaml). * - * Warnings flag IR content that has no loadable representation (an association - * relationship's junction detail), so the caller can surface the lossy edge. + * Warnings flag IR content that has no loadable representation, so the caller + * can surface the lossy edge. * * `logical` marks the model as a purely logical one (no physical binding), so * the missing-source and missing-expression warnings -- which flag a lossy pull @@ -309,46 +310,33 @@ function executorDoc(ex: Executor): Record { } // Inverts loader.convertRelationship: `from`/`to` are the endpoint entities and -// `from_columns`/`to_columns` are their positional join columns. A many-to-many -// edge instead carries an `association` block and, per the format, no join -// columns of its own -- the columns that bind it are on the junction table. +// `from_columns`/`to_columns` are the columns that reach them. A many-to-many +// edge adds the table it runs `through` -- the table those columns are on -- +// plus the key and properties that table gives it. +// +// `keys` is always written even though the format lets it be omitted: the +// loader's default is derived from the two column lists, and re-deriving it on +// the way out would silently rewrite an edge whose authored key differed from +// that default. function relationshipDoc( rel: Relationship, warnings: string[], logical: boolean): Record { dropExtensions(rel.customExtensions, `relationship '${rel.name}'`, warnings); - const association = rel.association ? - associationDoc(rel.association, rel.name, warnings, logical) : - undefined; return compact({ name: rel.name, from: rel.source.entity, to: rel.destination.entity, - from_columns: association ? undefined : nonEmpty(rel.source.columns), - to_columns: association ? undefined : nonEmpty(rel.destination.columns), - association, + through: rel.through, + keys: rel.through ? nonEmpty(rel.keys ?? []) : undefined, + from_columns: nonEmpty(rel.source.columns), + to_columns: nonEmpty(rel.destination.columns), + fields: + nonEmpty((rel.fields ?? []).map(f => fieldDoc(f, warnings, logical))), description: rel.description, ai_context: aiContextDoc(rel.aiContext), }); } -// Inverts loader.convertAssociation. `keys` is always written even though the -// format lets it be omitted: the loader's default is derived from the two -// column lists, and re-deriving it on the way out would silently rewrite an -// edge whose authored key differed from that default. -function associationDoc( - assoc: Association, relName: string, warnings: string[], - logical: boolean): Record { - return compact({ - source: assoc.dataSource, - keys: nonEmpty(assoc.keys), - from_columns: nonEmpty(assoc.sourceColumns), - to_columns: nonEmpty(assoc.destinationColumns), - fields: nonEmpty( - (assoc.fields ?? []) - .map(f => fieldDoc(f, warnings, logical))), - }); -} - // Renders the `expression` object matching the loader's schema (a `dialects` // array of {dialect, expression}). Emits the target/canonical form under // CANONICAL_DIALECT and the imported vendor form under its own dialect, so the diff --git a/toolbox/mdcode/src/libts/semantic/pull_kc.ts b/toolbox/mdcode/src/libts/semantic/pull_kc.ts index 26ed9500..c86303b8 100644 --- a/toolbox/mdcode/src/libts/semantic/pull_kc.ts +++ b/toolbox/mdcode/src/libts/semantic/pull_kc.ts @@ -20,8 +20,8 @@ import {CatalogClient, Entry, EntryLink} from '../gcp/dataplex'; import {SemanticModel} from './ir'; import {actionAspectTypes} from './kc_actions'; -import {associationAspectTypes} from './kc_associations'; -import {ACTION_TYPE_ID, ASSOCIATION_TYPE_ID} from './kc_custom_types'; +import {relationshipAspectTypes} from './kc_relationships'; +import {ACTION_TYPE_ID, RELATIONSHIP_TYPE_ID} from './kc_custom_types'; import {idOf, linkDedupKey, modelsFromCatalogResources} from './kc_converter'; export interface KcPullOptions { @@ -166,10 +166,10 @@ function semanticAspectTypes(entryType: string): string[]|undefined { // `typeBase` already points there, and kc_actions.ts names the aspects // to fetch beneath it. return actionAspectTypes(typeBase); - case ASSOCIATION_TYPE_ID: - // Likewise for a many-to-many relationship: its entry type is custom, and - // kc_associations.ts names the one aspect that holds the junction. - return associationAspectTypes(typeBase); + case RELATIONSHIP_TYPE_ID: + // Likewise for a relationship: its entry type is custom, and + // kc_relationships.ts names the one aspect that holds the whole edge. + return relationshipAspectTypes(typeBase); default: return undefined; } diff --git a/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts b/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts index 36043b5d..5858c378 100644 --- a/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts +++ b/toolbox/mdcode/src/libts/semantic/resolve_profiles.ts @@ -410,12 +410,14 @@ function firstUnboundReferenced( // The first join field of a relationship that is unbound on its own end, or // null when both ends' join columns are bound. // -// A many-to-many edge has no columns on either end -- the columns that bind it -// are on its junction table, which a profile does not reach -- so it is never -// dropped here. It still falls with an endpoint: unbinding an entity's key -// makes that entity unavailable, and the loop above drops every edge touching -// it. +// An edge with a `through` table is never dropped here: its join columns are on +// that table rather than on either endpoint, so they are not this model's +// fields and a profile cannot unbind them. Reading them as `Entity.column` would +// be a false match against a same-named field of the endpoint. Such an edge +// still falls with an endpoint: unbinding an entity's key makes that entity +// unavailable, and the loop above drops every edge touching it. function unboundJoinField(r: Relationship, unbound: Set): string|null { + if (r.through) return null; for (const c of r.source?.columns ?? []) { const key = `${r.source.entity}.${c}`; if (unbound.has(key)) return key; diff --git a/toolbox/mdcode/src/libts/semantic/spanner.ts b/toolbox/mdcode/src/libts/semantic/spanner.ts index 1c3355ca..754c2193 100644 --- a/toolbox/mdcode/src/libts/semantic/spanner.ts +++ b/toolbox/mdcode/src/libts/semantic/spanner.ts @@ -3,7 +3,7 @@ // The IR (./ir) is pure semantics. This module is one of its consumers, a // sibling to ./bigquery: it emits a single `CREATE OR REPLACE PROPERTY GRAPH` // statement over the entities' existing Spanner input tables. It shares the IR, -// the inheritance-resolution pass, and the edge/association shapes with the +// the inheritance-resolution pass, and the edge shapes with the // BigQuery generator, but differs from it in three ways that follow from // Spanner Graph's DDL: // @@ -23,7 +23,7 @@ // https://docs.cloud.google.com/spanner/docs/reference/standard-sql/graph-schema-statements // -import {Association, Entity, Field, Relationship, SemanticModel} from './ir'; +import {Entity, Field, Relationship, SemanticModel} from './ir'; import {resolveInheritance} from './resolve_inheritance'; import {stripQualifier} from './sql_expr_utils'; import {isSimpleIdentifier, quoteIdentifier, quoteIfReserved} from './sql_identifiers'; @@ -246,9 +246,8 @@ function renderNodeTable( function renderEdgeTable( rel: Relationship, entitiesByName: Map, warnings: string[]): string { - if (rel.association) { - return renderAssociationEdge( - rel, rel.association, entitiesByName, warnings); + if (rel.through) { + return renderThroughEdge(rel, rel.through, entitiesByName, warnings); } // A direct foreign key: the SOURCE entity's own table backs the edge (one // edge row per source row). Its FK columns reference the destination's key @@ -299,19 +298,19 @@ function renderEdgeTable( } -// Renders a many-to-many edge backed by an association (junction) table. The -// edge has its OWN backing table and KEY, each endpoint's SOURCE/DESTINATION -// KEY names the junction columns referencing that entity's declared key, and -// the junction's own `fields` become edge PROPERTIES. -function renderAssociationEdge( - rel: Relationship, assoc: Association, entitiesByName: Map, +// Renders a many-to-many edge, which runs through a table of its own. The edge +// has its OWN backing table and KEY, each endpoint's SOURCE/DESTINATION KEY +// names the columns on that table referencing the entity's declared key, and +// the edge's own `fields` become PROPERTIES. +function renderThroughEdge( + rel: Relationship, through: string, entitiesByName: Map, warnings: string[]): string { const backing = - spannerTable(assoc.dataSource, warnings, `relationship '${rel.name}'`); - if (!assoc.keys?.length) { + spannerTable(through, warnings, `relationship '${rel.name}'`); + if (!rel.keys?.length) { warnings.push( - `relationship '${rel.name}': association table has no KEY; the edge ` + - `table will be invalid (an edge requires a KEY)`); + `relationship '${rel.name}': the table it runs through has no KEY; ` + + `the edge table will be invalid (an edge requires a KEY)`); } const refColumns = (end: {entity: string; columns: string[]}): string => { @@ -328,22 +327,22 @@ function renderAssociationEdge( const lines = [ line(1, `${backing} AS ${quoteIfReserved(rel.name)}`), - line(2, `KEY(${assoc.keys.map(quoteIfReserved).join(', ')})`), + line(2, `KEY(${(rel.keys ?? []).map(quoteIfReserved).join(', ')})`), line( 2, - `SOURCE KEY(${assoc.sourceColumns.map(quoteIfReserved).join(', ')}) ` + + `SOURCE KEY(${rel.source.columns.map(quoteIfReserved).join(', ')}) ` + `REFERENCES ${quoteIfReserved(rel.source.entity)}(${ refColumns(rel.source)})`), line( 2, `DESTINATION KEY(${ - assoc.destinationColumns.map(quoteIfReserved).join(', ')}) ` + + rel.destination.columns.map(quoteIfReserved).join(', ')}) ` + `REFERENCES ${quoteIfReserved(rel.destination.entity)}(${ refColumns(rel.destination)})`), ]; const properties = - (assoc.fields ?? []).map(f => renderFieldProperty(f, rel.name)); + (rel.fields ?? []).map(f => renderFieldProperty(f, rel.name)); if (properties.length) lines.push(propertiesBlock(properties)); return lines.join('\n'); diff --git a/toolbox/mdcode/src/libts/semantic/transpile.ts b/toolbox/mdcode/src/libts/semantic/transpile.ts index 748878db..46f7d251 100644 --- a/toolbox/mdcode/src/libts/semantic/transpile.ts +++ b/toolbox/mdcode/src/libts/semantic/transpile.ts @@ -140,10 +140,10 @@ export async function transpileModel( for (const f of e.fields) add(f, `field '${e.name}.${f.name}'`); } for (const r of clone.relationships) { - // Direct FK edges carry no expressions; only an association (junction + // Direct FK edges carry no expressions; only a through-edge (a table // table) has edge-property fields, and only when hand-built IR supplies // them. - for (const f of r.association?.fields ?? []) { + for (const f of r.fields ?? []) { add(f, `relationship '${r.name}' field '${f.name}'`); } } diff --git a/toolbox/mdcode/src/libts/semantic/validate.ts b/toolbox/mdcode/src/libts/semantic/validate.ts index 91d19cef..35f14a42 100644 --- a/toolbox/mdcode/src/libts/semantic/validate.ts +++ b/toolbox/mdcode/src/libts/semantic/validate.ts @@ -95,11 +95,9 @@ export function validatePushRequirements( // target (both arrays empty), so this is skipped. if (deployInfo.bigQuery.length + deployInfo.spanner.length > 0) { for (const rel of model.relationships ?? []) { - // An M:N edge binds through its junction table (association), so its - // direct source/destination columns are empty by design -- bigquery.ts - // renders it from `rel.association`. Only a plain FK edge needs direct - // join columns. - if (rel.association) continue; + // Both edge shapes bind with join columns -- a foreign-key edge's are + // on the endpoints' own tables, a through-edge's are on the table it + // runs through -- so neither is exempt from having them. if (!rel.source.columns.length || !rel.destination.columns.length) { errors.push( `relationship '${rel.name}' in model '${model.name}' (${ diff --git a/toolbox/mdcode/tests/libts/semantic/bigquery.test.ts b/toolbox/mdcode/tests/libts/semantic/bigquery.test.ts index 0c4648c7..de8572dc 100644 --- a/toolbox/mdcode/tests/libts/semantic/bigquery.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/bigquery.test.ts @@ -9,9 +9,9 @@ // // This file holds only what a loader fixture CANNOT express, because the open // AI-first format the loader reads is a subset of the IR: -// - an M:N association edge (its own backing junction table, KEY, and edge -// properties) — the open format has no association-table syntax, so its IR -// is hand-built here and checked against a committed golden file. +// - a through-edge whose alias, KEY, join columns, and REFERENCES labels are +// all reserved words — the loader would reject those names, so that IR is +// hand-built here and checked against a committed golden file. // - IR-contract cases the loader never produces: a COUNT(*) metric with a // declared attach entity, and a metric whose declared entity disagrees with // its expression (the loader always derives the entity FROM the @@ -48,10 +48,10 @@ const GEN_OPTS: GenerateOptions = { }; -describe('M:N association edge', () => { - // Loaded from `school_manytomany.yaml`, which authors the junction table with - // the extended profile's `association` block, so this covers the whole path - // from the format to the DDL. The expected DDL is a committed golden file +describe('many-to-many edge', () => { + // Loaded from `school_manytomany.yaml`, which authors the table the edge runs + // through with the extended profile's `through` key, so this covers the whole + // path from the format to the DDL. The expected DDL is a committed golden file // (`school_manytomany.bigquery.golden.sql`) so the output stays reviewable as // text; these exact strings were run against a live BigQuery instance and // traversed with a GQL MATCH. @@ -61,7 +61,7 @@ describe('M:N association edge', () => { dataset: 'bei_semantic_ir_verify' }; - test('the association graph matches its committed golden DDL', () => { + test('the many-to-many graph matches its committed golden DDL', () => { const {ddl} = generatePropertyGraph(SCHOOL, SCHOOL_OPTS); const golden = fs.readFileSync( path.join(FIXTURES, 'school_manytomany.bigquery.golden.sql'), 'utf8'); @@ -898,11 +898,11 @@ describe('inherited property rendering (shared-label consistency)', () => { }); }); -describe('reserved-word names in an M:N association edge are quoted', () => { - // The open format has no association-table syntax, so this hand-built IR is - // the only path that exercises renderAssociationEdge's identifier quoting: the - // edge alias, KEY, SOURCE KEY / DESTINATION KEY columns, and both REFERENCES - // labels, each named with a GoogleSQL reserved keyword. +describe('reserved-word names in a through-edge are quoted', () => { + // The loader rejects reserved-word names, so this hand-built IR is the only + // path that exercises renderThroughEdge's identifier quoting: the edge alias, + // KEY, SOURCE KEY / DESTINATION KEY columns, and both REFERENCES labels, each + // named with a GoogleSQL reserved keyword. const RW_ASSOC: SemanticModel = { name: 'rw_assoc', entities: [ @@ -923,13 +923,8 @@ describe('reserved-word names in an M:N association edge are quoted', () => { name: 'from', source: {entity: 'Order', columns: ['order']}, destination: {entity: 'Group', columns: ['id']}, - association: { - dataSource: 'proj.ds.order_group', - keys: ['order'], - sourceColumns: ['order'], - destinationColumns: ['id'], - fields: [], - }, + through: 'proj.ds.order_group', + keys: ['order'], }], metrics: [], }; diff --git a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts index 9ad830ab..4e10f5a0 100644 --- a/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/deploy_knowledge_catalog.test.ts @@ -216,9 +216,10 @@ describe('deployKnowledgeCatalog: relationship entry links', () => { const result = await deployKnowledgeCatalog(models(STAR_DOCS), CTX, OPTS); expect(result.success).toBe(true); - expect(result.created).toBe(5); // anchor + 2 entities + 2 metrics + // anchor + 2 entities + 2 metrics + 1 relationship + expect(result.created).toBe(6); expect(result.linked).toBe(1); - expect(create).toHaveBeenCalledTimes(5); + expect(create).toHaveBeenCalledTimes(6); expect(createLink).toHaveBeenCalledTimes(1); expect(updateLink).not.toHaveBeenCalled(); // Links are written to the same destination the entries are. diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.knowledge_catalog.golden.json index a7f59c87..e6602429 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.knowledge_catalog.golden.json +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.knowledge_catalog.golden.json @@ -99,19 +99,19 @@ } }, { - "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph.associations.enrollment", - "entryType": "projects/sqlgen-testing/locations/global/entryTypes/semantic-association", + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph.relationships.enrollment", + "entryType": "projects/sqlgen-testing/locations/global/entryTypes/semantic-relationship", "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/school_graph", "entrySource": { "displayName": "enrollment" }, "aspects": { - "sqlgen-testing.global.semantic-association": { - "aspectType": "projects/sqlgen-testing/locations/global/aspectTypes/semantic-association", + "sqlgen-testing.global.semantic-relationship": { + "aspectType": "projects/sqlgen-testing/locations/global/aspectTypes/semantic-relationship", "data": { "fromEntity": "students", "toEntity": "courses", - "junction": "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/bei_semantic_ir_verify/tables/enrollment", + "through": "//bigquery.googleapis.com/projects/sqlgen-testing/datasets/bei_semantic_ir_verify/tables/enrollment", "keys": [ "enrollment_id" ], diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.osi.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.osi.golden.yaml index 95e49a68..35676a78 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.osi.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.osi.golden.yaml @@ -37,18 +37,17 @@ semantic_model: - name: enrollment from: students to: courses - association: - source: sqlgen-testing.bei_semantic_ir_verify.enrollment - keys: - - enrollment_id - from_columns: - - student_id - to_columns: - - course_id - fields: - - name: grade - expression: - dialects: - - dialect: BIGQUERY - expression: enrollment.grade - description: Letter grade + through: sqlgen-testing.bei_semantic_ir_verify.enrollment + keys: + - enrollment_id + from_columns: + - student_id + to_columns: + - course_id + fields: + - name: grade + expression: + dialects: + - dialect: BIGQUERY + expression: enrollment.grade + description: Letter grade diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.pull.golden.yaml index d232ccff..3fee0943 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.pull.golden.yaml @@ -26,19 +26,18 @@ semantic_model: - name: enrollment from: students to: courses - association: - source: sqlgen-testing.bei_semantic_ir_verify.enrollment - keys: - - enrollment_id - from_columns: - - student_id - to_columns: - - course_id - fields: - - name: grade - expression: - dialects: - - dialect: BIGQUERY - expression: enrollment.grade - datatype: Opaque - description: Letter grade + through: sqlgen-testing.bei_semantic_ir_verify.enrollment + keys: + - enrollment_id + from_columns: + - student_id + to_columns: + - course_id + fields: + - name: grade + expression: + dialects: + - dialect: BIGQUERY + expression: enrollment.grade + datatype: Opaque + description: Letter grade diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.yaml index 905e61a0..daccd2cc 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/school_manytomany.yaml @@ -2,10 +2,10 @@ # taken by many students, so the pairs live in an `enrollment` table of their # own rather than in a foreign key on either side. # -# The `association` block is a native key of the extended profile, so this -# document declares `0.2.0.dev0/google`. Its BigQuery and Spanner DDL are the -# committed `school_manytomany.*.golden.sql` goldens, which were run against a -# live BigQuery instance and traversed with a GQL MATCH. +# `through` (and the `keys`/`fields` it brings) is a native key of the extended +# profile, so this document declares `0.2.0.dev0/google`. Its BigQuery and +# Spanner DDL are the committed `school_manytomany.*.golden.sql` goldens, which +# were run against a live BigQuery instance and traversed with a GQL MATCH. version: "0.2.0.dev0/google" @@ -47,19 +47,18 @@ semantic_model: - name: enrollment from: students to: courses - # No from_columns/to_columns on the relationship itself: neither student - # nor course holds a foreign key. The columns that bind the edge are on - # the junction table below. - association: - source: sqlgen-testing.bei_semantic_ir_verify.enrollment - keys: [enrollment_id] - from_columns: [student_id] - to_columns: [course_id] - # A property of the pairing, not of either endpoint. - fields: - - name: grade - expression: - dialects: - - dialect: BIGQUERY - expression: enrollment.grade - description: Letter grade + # The edge runs through the enrollment table, so from_columns and + # to_columns are columns on THAT table, not on students or courses -- + # neither endpoint holds a foreign key. + through: sqlgen-testing.bei_semantic_ir_verify.enrollment + keys: [enrollment_id] + from_columns: [student_id] + to_columns: [course_id] + # A property of the pairing, not of either endpoint. + fields: + - name: grade + expression: + dialects: + - dialect: BIGQUERY + expression: enrollment.grade + description: Letter grade diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json index 66e5aa95..789103be 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.knowledge_catalog.golden.json @@ -160,6 +160,29 @@ } } } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales.relationships.orders_to_customer", + "entryType": "projects/sqlgen-testing/locations/global/entryTypes/semantic-relationship", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/sales", + "entrySource": { + "displayName": "orders_to_customer" + }, + "aspects": { + "sqlgen-testing.global.semantic-relationship": { + "aspectType": "projects/sqlgen-testing/locations/global/aspectTypes/semantic-relationship", + "data": { + "fromEntity": "orders", + "toEntity": "customer", + "fromColumns": [ + "o_custkey" + ], + "toColumns": [ + "c_custkey" + ] + } + } + } } ], "entryLinks": [ @@ -204,7 +227,5 @@ } } ], - "warnings": [ - "relationship 'orders_to_customer': Knowledge Catalog stores the name only in the normalized link id, so a pull returns it lowercased/hyphenated (e.g. 'orders-to-customer'), not 'orders_to_customer'." - ] + "warnings": [] } diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml index ba997f2b..dc7c0bac 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/star_orders_customer.pull.golden.yaml @@ -34,7 +34,7 @@ semantic_model: datatype: Opaque description: Customer name relationships: - - name: orders-to-customer + - name: orders_to_customer from: orders to: customer from_columns: diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json index e9d496d4..8fc82c30 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.knowledge_catalog.golden.json @@ -281,6 +281,98 @@ } } } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.relationships.store_sales_to_date_dim", + "entryType": "projects/sqlgen-testing/locations/global/entryTypes/semantic-relationship", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "store_sales_to_date_dim" + }, + "aspects": { + "sqlgen-testing.global.semantic-relationship": { + "aspectType": "projects/sqlgen-testing/locations/global/aspectTypes/semantic-relationship", + "data": { + "fromEntity": "store_sales", + "toEntity": "date_dim", + "fromColumns": [ + "ss_sold_date_sk" + ], + "toColumns": [ + "ss_sold_date_sk" + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.relationships.store_sales_to_customer", + "entryType": "projects/sqlgen-testing/locations/global/entryTypes/semantic-relationship", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "store_sales_to_customer" + }, + "aspects": { + "sqlgen-testing.global.semantic-relationship": { + "aspectType": "projects/sqlgen-testing/locations/global/aspectTypes/semantic-relationship", + "data": { + "fromEntity": "store_sales", + "toEntity": "customer", + "fromColumns": [ + "ss_customer_sk" + ], + "toColumns": [ + "c_customer_sk" + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.relationships.store_sales_to_item", + "entryType": "projects/sqlgen-testing/locations/global/entryTypes/semantic-relationship", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "store_sales_to_item" + }, + "aspects": { + "sqlgen-testing.global.semantic-relationship": { + "aspectType": "projects/sqlgen-testing/locations/global/aspectTypes/semantic-relationship", + "data": { + "fromEntity": "store_sales", + "toEntity": "item", + "fromColumns": [ + "ss_item_sk" + ], + "toColumns": [ + "i_item_sk" + ] + } + } + } + }, + { + "name": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model.relationships.store_sales_to_store", + "entryType": "projects/sqlgen-testing/locations/global/entryTypes/semantic-relationship", + "parentEntry": "projects/sqlgen-testing/locations/us/entryGroups/semantic/entries/tpcds_model", + "entrySource": { + "displayName": "store_sales_to_store" + }, + "aspects": { + "sqlgen-testing.global.semantic-relationship": { + "aspectType": "projects/sqlgen-testing/locations/global/aspectTypes/semantic-relationship", + "data": { + "fromEntity": "store_sales", + "toEntity": "store", + "fromColumns": [ + "ss_store_sk" + ], + "toColumns": [ + "s_store_sk" + ] + } + } + } } ], "entryLinks": [ @@ -446,10 +538,6 @@ } ], "warnings": [ - "entity 'date_dim': no keys declared in the source model", - "relationship 'store_sales_to_date_dim': Knowledge Catalog stores the name only in the normalized link id, so a pull returns it lowercased/hyphenated (e.g. 'store-sales-to-date-dim'), not 'store_sales_to_date_dim'.", - "relationship 'store_sales_to_customer': Knowledge Catalog stores the name only in the normalized link id, so a pull returns it lowercased/hyphenated (e.g. 'store-sales-to-customer'), not 'store_sales_to_customer'.", - "relationship 'store_sales_to_item': Knowledge Catalog stores the name only in the normalized link id, so a pull returns it lowercased/hyphenated (e.g. 'store-sales-to-item'), not 'store_sales_to_item'.", - "relationship 'store_sales_to_store': Knowledge Catalog stores the name only in the normalized link id, so a pull returns it lowercased/hyphenated (e.g. 'store-sales-to-store'), not 'store_sales_to_store'." + "entity 'date_dim': no keys declared in the source model" ] } diff --git a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml index 9173162a..09cd9060 100644 --- a/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml +++ b/toolbox/mdcode/tests/libts/semantic/fixtures/tpcds_date_edge.pull.golden.yaml @@ -81,28 +81,28 @@ semantic_model: source: sqlgen-testing.demo.date_dim description: Date dimension with calendar attributes relationships: - - name: store-sales-to-date-dim + - name: store_sales_to_date_dim from: store_sales to: date_dim from_columns: - ss_sold_date_sk to_columns: - ss_sold_date_sk - - name: store-sales-to-customer + - name: store_sales_to_customer from: store_sales to: customer from_columns: - ss_customer_sk to_columns: - c_customer_sk - - name: store-sales-to-item + - name: store_sales_to_item from: store_sales to: item from_columns: - ss_item_sk to_columns: - i_item_sk - - name: store-sales-to-store + - name: store_sales_to_store from: store_sales to: store from_columns: diff --git a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts index b9bd050d..0637d29f 100644 --- a/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/kc_converter.test.ts @@ -26,7 +26,7 @@ import * as fs from 'node:fs'; import * as path from 'node:path'; import {CustomExtension, Entity, Metric, SemanticModel} from '../../../src/libts/semantic/ir'; -import {linkNamePrefix, modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; +import {modelsFromCatalogResources} from '../../../src/libts/semantic/kc_converter'; import {generateCatalogResources} from '../../../src/libts/semantic/knowledge_catalog'; import {loadModels} from '../../../src/libts/semantic/loader'; import {serializeModel} from '../../../src/libts/semantic/osi_converter'; @@ -60,9 +60,8 @@ function roundTripFull(model: SemanticModel): describe('emitter -> reader round trip (lossless slice)', () => { - // A model using only round-trippable content: no relationships (dropped when - // M:N, else name-normalized) and no still-lossy ai_context; datatypes that - // invert cleanly. Its fields and metric carry expressions and a DIMENSION + // A model using only round-trippable content: no relationships and no + // still-lossy ai_context; datatypes that invert cleanly. Its fields and metric carry expressions and a DIMENSION // role, so it round-trips losslessly only through a `--emit-expressions` push // (roundTripFull); a default push omits the per-field `semantics` block. const source: SemanticModel = { @@ -225,7 +224,7 @@ describe('a guidelines aspect is recovered only when author-managed', () => { }); -describe('relationship recovery (schema-join links -> IR)', () => { +describe('relationship recovery (semantic-relationship entries -> IR)', () => { // orders.o_custkey (the foreign-key side) references customer.c_custkey. const twoEntities: Entity[] = [ { @@ -242,6 +241,18 @@ describe('relationship recovery (schema-join links -> IR)', () => { }, ]; + // A read of the same push with the relationship entries removed, leaving only + // the schema-join links. That is the shape a catalog written by an older kcmd + // has, and pull falls back to the links for it, so the cases below that are + // about the link path go through this rather than roundTrip. + function viaLinksOnly(model: SemanticModel): + {models: SemanticModel[]; warnings: string[]} { + const {entries, entryLinks} = generateCatalogResources(model, OPTS); + return modelsFromCatalogResources( + entries.filter(e => !e.entryType.endsWith('/semantic-relationship')), + entryLinks); + } + test( 'a 1:N relationship recovers its endpoints, direction, and columns', () => { @@ -249,8 +260,7 @@ describe('relationship recovery (schema-join links -> IR)', () => { name: 'sales', entities: twoEntities, relationships: [{ - name: - 'places', // already link-slug-safe, so it round-trips exactly + name: 'places', source: {entity: 'orders', columns: ['o_custkey']}, destination: {entity: 'customer', columns: ['c_custkey']}, }], @@ -264,44 +274,94 @@ describe('relationship recovery (schema-join links -> IR)', () => { }]); }); - test( - 'a relationship name comes back normalized (lowercased/hyphenated)', - () => { - const model: SemanticModel = { - name: 'sales', - entities: twoEntities, - relationships: [{ - name: 'Places_Order', // mixed case + underscore -> normalized on - // read - source: {entity: 'orders', columns: ['o_custkey']}, - destination: {entity: 'customer', columns: ['c_custkey']}, - }], - metrics: [], - }; - expect(roundTrip(model).models[0].relationships[0].name) - .toBe('places-order'); - }); + test('a relationship name comes back verbatim, whatever its casing', () => { + // The entry carries the authored name in its displayName, so mixed case and + // underscores survive. Read from the link alone they would not (see below): + // the link has no name field, only an id. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'Places_Order', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].name) + .toBe('Places_Order'); + }); - test('a many-to-many relationship round-trips through its own entry', () => { - // It is not a schema-join link but a semantic-association entry, so it - // comes back from `entries` rather than `entryLinks` -- and keeps its - // authored name, which the entry carries verbatim. + test('read from the link alone, a name comes back normalized', () => { + // The fallback path for a catalog written before relationships had entries: + // the name is reconstructed from the link id, which is lowercased and + // hyphenated. const model: SemanticModel = { name: 'sales', entities: twoEntities, relationships: [{ - name: 'Promoted_By', + name: 'Places_Order', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + }], + metrics: [], + }; + expect(viaLinksOnly(model).models[0].relationships[0].name) + .toBe('places-order'); + }); + + test('a purely logical relationship round-trips', () => { + // No columns on either end, so there is no schema-join link to carry it; + // the entry is its only route through the catalog. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'Reached', source: {entity: 'orders', columns: []}, destination: {entity: 'customer', columns: []}, description: 'which customers an order reached', - association: { - dataSource: 'p.d.junction', - keys: ['id'], - sourceColumns: ['j_orderkey'], - destinationColumns: ['j_custkey'], - fields: [{name: 'discount', expression: 'j.discount', - type: 'Decimal'}], - }, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships).toEqual([ + model.relationships[0] + ]); + }); + + test('relationship instructions round-trip', () => { + // ai_context on a relationship rides the relationship aspect; before the + // entry existed it had nowhere in the catalog to live. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'places', + source: {entity: 'orders', columns: ['o_custkey']}, + destination: {entity: 'customer', columns: ['c_custkey']}, + aiContext: {instructions: 'Join through here for a customer of record.'}, + }], + metrics: [], + }; + expect(roundTrip(model).models[0].relationships[0].aiContext) + .toEqual({instructions: 'Join through here for a customer of record.'}); + }); + + test('a many-to-many relationship round-trips whole', () => { + // The table it runs through, that table's key, the columns on it, and the + // properties of the pairing all ride the relationship aspect. + const model: SemanticModel = { + name: 'sales', + entities: twoEntities, + relationships: [{ + name: 'Promoted_By', + source: {entity: 'orders', columns: ['j_orderkey']}, + destination: {entity: 'customer', columns: ['j_custkey']}, + through: 'p.d.junction', + keys: ['id'], + fields: [{name: 'discount', expression: 'j.discount', + type: 'Decimal'}], + description: 'which customers an order reached', }], metrics: [], }; @@ -321,75 +381,62 @@ describe('relationship recovery (schema-join links -> IR)', () => { destination: {entity: 'customer', columns: ['c_custkey']}, }, { - name: 'promoted-by', - source: {entity: 'orders', columns: []}, - destination: {entity: 'customer', columns: []}, - association: { - dataSource: 'p.d.junction', - keys: [], - sourceColumns: ['j_orderkey'], - destinationColumns: ['j_custkey'], - }, + name: 'promoted_by', + source: {entity: 'orders', columns: ['j_orderkey']}, + destination: {entity: 'customer', columns: ['j_custkey']}, + through: 'p.d.junction', + keys: [], }, ], metrics: [], }; expect(roundTrip(model).models[0].relationships.map(r => r.name)) - .toEqual(['places', 'promoted-by']); + .toEqual(['places', 'promoted_by']); }); - test('an association entry naming an unknown entity is skipped', () => { + test('a relationship entry naming an unknown entity is skipped', () => { const model: SemanticModel = { name: 'sales', entities: twoEntities, relationships: [{ - name: 'promoted-by', - source: {entity: 'orders', columns: []}, - destination: {entity: 'customer', columns: []}, - association: { - dataSource: 'p.d.junction', - keys: [], - sourceColumns: ['j_orderkey'], - destinationColumns: ['j_custkey'], - }, + name: 'promoted_by', + source: {entity: 'orders', columns: ['j_orderkey']}, + destination: {entity: 'customer', columns: ['j_custkey']}, + through: 'p.d.junction', + keys: [], }], metrics: [], }; const {entries, entryLinks} = generateCatalogResources(model, OPTS); - const assoc = entries.find( - e => e.entryType.endsWith('/entryTypes/semantic-association'))!; - assoc.aspects!['dest.global.semantic-association'].data!.toEntity = - 'ghost'; + const rel = entries.find( + e => e.entryType.endsWith('/entryTypes/semantic-relationship'))!; + rel.aspects!['dest.global.semantic-relationship'].data!.toEntity = 'ghost'; const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); expect(models[0].relationships).toEqual([]); expect(warnings.some(w => w.includes('ghost'))).toBe(true); }); - test('an association entry with no junction columns is skipped', () => { + test('a through table with no columns on it is skipped', () => { const model: SemanticModel = { name: 'sales', entities: twoEntities, relationships: [{ - name: 'promoted-by', - source: {entity: 'orders', columns: []}, - destination: {entity: 'customer', columns: []}, - association: { - dataSource: 'p.d.junction', - keys: [], - sourceColumns: ['j_orderkey'], - destinationColumns: ['j_custkey'], - }, + name: 'promoted_by', + source: {entity: 'orders', columns: ['j_orderkey']}, + destination: {entity: 'customer', columns: ['j_custkey']}, + through: 'p.d.junction', + keys: [], }], metrics: [], }; const {entries, entryLinks} = generateCatalogResources(model, OPTS); - const assoc = entries.find( - e => e.entryType.endsWith('/entryTypes/semantic-association'))!; - delete assoc.aspects!['dest.global.semantic-association'] - .data!.toColumns; + const rel = entries.find( + e => e.entryType.endsWith('/entryTypes/semantic-relationship'))!; + delete rel.aspects!['dest.global.semantic-relationship'].data!.toColumns; const {models, warnings} = modelsFromCatalogResources(entries, entryLinks); expect(models[0].relationships).toEqual([]); - expect(warnings.some(w => w.includes('no junction column'))).toBe(true); + expect(warnings.some(w => w.includes("'through' table but no column"))) + .toBe(true); }); test( @@ -423,7 +470,7 @@ describe('relationship recovery (schema-join links -> IR)', () => { }], metrics: [], }; - const {models, warnings} = roundTrip(model); + const {models, warnings} = viaLinksOnly(model); expect(models[0].relationships).toEqual([{ name: 'parents', source: {entity: 'child_order', columns: ['parent_id']}, @@ -463,7 +510,10 @@ describe('relationship recovery (schema-join links -> IR)', () => { 'projects/dest/', 'projects/000000000000/'), })), })); - const {models} = modelsFromCatalogResources(entries, numeric); + const {models} = modelsFromCatalogResources( + entries.filter( + e => !e.entryType.endsWith('/semantic-relationship')), + numeric); expect(models[0].relationships).toEqual([{ name: 'places', source: {entity: 'orders', columns: ['o_custkey']}, @@ -489,7 +539,7 @@ describe('relationship recovery (schema-join links -> IR)', () => { }], metrics: [], }; - expect(roundTrip(model).models[0].relationships[0].name) + expect(viaLinksOnly(model).models[0].relationships[0].name) .toBe('rel-one'); } }); @@ -511,7 +561,8 @@ describe('relationship recovery (schema-join links -> IR)', () => { const {entries, entryLinks} = generateCatalogResources(model, OPTS); const nameless = entryLinks.map(l => ({...l, name: undefined})); const {models} = modelsFromCatalogResources( - entries, [...nameless, ...nameless.map(l => ({...l}))]); + entries.filter(e => !e.entryType.endsWith('/semantic-relationship')), + [...nameless, ...nameless.map(l => ({...l}))]); expect(models[0].relationships).toHaveLength(1); }); }); @@ -984,32 +1035,23 @@ function stripToKcFloor(model: SemanticModel): SemanticModel { } } - // The two arities lose different things, because they are published as - // different resources. A direct foreign key is a schema-join entry LINK, - // which carries no name field and no aspect for guidelines, so the name comes - // back normalized via the emitter's slug and ai_context is gone. A - // many-to-many edge is an ENTRY of the custom semantic-association type, - // which carries its own display name and folds instructions and the edge's - // fields into its own aspect, so all three survive. + // Every relationship, whatever its arity, is published as an ENTRY of the + // custom semantic-relationship type. The entry carries its own display name + // and folds instructions and the edge's own fields into its own aspect, so + // the name survives verbatim and ai_context survives down to instructions. + // Only the vendor extension blocks have no home. m.relationships = m.relationships.map(r => { const rel = structuredClone(r); delete rel.customExtensions; - if (!rel.association) { - rel.name = linkNamePrefix(rel.name); - delete rel.aiContext; - return rel; - } floorAiContext(rel); // An empty field list is written as nothing and reads back absent. - if (rel.association.fields && !rel.association.fields.length) { - delete rel.association.fields; - } - for (const f of rel.association.fields ?? []) { + if (rel.fields && !rel.fields.length) delete rel.fields; + for (const f of rel.fields ?? []) { delete f.aiContext; delete f.importedExpression; delete f.importedDialect; delete f.customExtensions; - // The association aspect has no slot for a display label or a dimension + // The relationship aspect has no slot for a display label or a dimension // role. It does store the field's expression -- unlike an entity field, // whose expression is gated off because the BUILT-IN schema template has // nowhere to put it; this template is ours, so it carries one. diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts index 6e23b46c..0e5c555e 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.e2e.test.ts @@ -30,9 +30,9 @@ const FIXTURES = path.join(__dirname, 'fixtures'); // Fixtures that get a KC golden. Chosen to exercise the distinct mappings: // sales_bq_graph_target -> model aspect deploymentTargets + un-typed metric // (dataType fallback); star_orders_customer -> a direct-FK relationship -// (schema-join link) + multiple entities/metrics; tpcds_date_edge -> -// temporal field types; school_manytomany -> a junction-backed edge, which -// is a semantic-association entry rather than a link. +// (entry + schema-join link) + multiple entities/metrics; tpcds_date_edge +// -> temporal field types; school_manytomany -> an edge through a table of +// pairs, which gets an entry and no link. const CORPUS = [ 'sales_bq_graph_target.yaml', 'star_orders_customer.yaml', diff --git a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts index dcdc1ecc..8294a8c0 100644 --- a/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/knowledge_catalog.test.ts @@ -496,32 +496,31 @@ describe('model-level structure', () => { }); -describe('relationships map to schema-join entry links', () => { - // A two-entity model joined by a direct foreign key (orders.custkey -> - // customer.custkey), the common case the loader produces from - // `relationships`. - function directFkModel(): SemanticModel { - return { - name: 'm', - metrics: [], - entities: [ - {name: 'orders', dataSource: 'p.d.orders', keys: ['o_key'], fields: []}, - { - name: 'customer', - dataSource: 'p.d.customer', - keys: ['c_key'], - fields: [] - }, - ], - relationships: [{ - name: 'orders-to-customer', - source: {entity: 'orders', columns: ['custkey']}, - destination: {entity: 'customer', columns: ['c_key']}, - description: 'each order belongs to a customer', - }], - }; - } +// A two-entity model joined by a direct foreign key (orders.custkey -> +// customer.custkey), the common case the loader produces from `relationships`. +function directFkModel(): SemanticModel { + return { + name: 'm', + metrics: [], + entities: [ + {name: 'orders', dataSource: 'p.d.orders', keys: ['o_key'], fields: []}, + { + name: 'customer', + dataSource: 'p.d.customer', + keys: ['c_key'], + fields: [] + }, + ], + relationships: [{ + name: 'orders-to-customer', + source: {entity: 'orders', columns: ['custkey']}, + destination: {entity: 'customer', columns: ['c_key']}, + description: 'each order belongs to a customer', + }], + }; +} +describe('relationships map to schema-join entry links', () => { test('a direct FK becomes one FOREIGN_KEY schema-join link', () => { const {entryLinks, warnings} = generateCatalogResources(directFkModel(), OPTS); @@ -578,44 +577,39 @@ describe('relationships map to schema-join entry links', () => { .toBe(true); }); - test('a column-less (purely logical) edge is skipped and warned, no link', - () => { - // An OWL import leaves an edge with no join columns. schema-join is a - // server CLOSED template, so rather than risk a rejected column-less - // aspect the emitter skips the link (the edge publishes once join - // columns are added to the model). - const model = directFkModel(); - model.relationships[0].source.columns = []; - model.relationships[0].destination.columns = []; - const {entryLinks, warnings} = generateCatalogResources(model, OPTS); - expect(entryLinks.length).toBe(0); - expect(warnings.some( - w => w.includes('orders-to-customer') && - w.includes('no join columns'))) - .toBe(true); - }); + test('a column-less (purely logical) edge produces no link, silently', () => { + // An OWL import leaves an edge with no join columns. schema-join is a + // server CLOSED template, so rather than risk a rejected column-less aspect + // the emitter skips the link. Silently: the edge is still published, as an + // entry, so nothing is lost by having no link for it. + const model = directFkModel(); + model.relationships[0].source.columns = []; + model.relationships[0].destination.columns = []; + const {entryLinks, warnings} = generateCatalogResources(model, OPTS); + expect(entryLinks.length).toBe(0); + expect(warnings.length).toBe(0); + }); - test( - 'a relationship name that is not link-id-clean warns it will normalize', - () => { - const model = directFkModel(); - // Underscores + uppercase are not valid in a link id, so the emitter - // slugs the name into the link id; a pull can only recover that slugged - // form. Warn so the author knows the round trip renames it. - model.relationships[0].name = 'Orders_To_Customer'; - const {entryLinks, warnings} = generateCatalogResources(model, OPTS); - expect(entryLinks.length).toBe(1); - expect(warnings.some( - w => w.includes('Orders_To_Customer') && - w.includes('orders-to-customer'))) - .toBe(true); - }); + test('a name that is not link-id-clean is slugged into the link id', () => { + // Underscores and uppercase are not valid in a link id, so the emitter + // slugs the name. No warning: the entry carries the authored name verbatim, + // and a pull reads the name from there, so the slug never reaches the + // author. + const model = directFkModel(); + model.relationships[0].name = 'Orders_To_Customer'; + const {entryLinks, warnings} = generateCatalogResources(model, OPTS); + expect(entryLinks.length).toBe(1); + expect(entryLinks[0].name!.endsWith('/entryLinks/m-orders-to-customer')) + .toBe(true); + expect(warnings.length).toBe(0); + }); }); -// The same two entities paired through a junction table instead of a foreign -// key: an order is on many promotions and a promotion covers many orders, so -// the pairs live in `p.d.order_promotion` with a `discount` of their own. +// The same two entities paired through a table of their own instead of a +// foreign key: an order reaches many customers and a customer is reached by +// many orders, so the pairs live in `p.d.order_customer` with a `discount` of +// their own. function mnModel(): SemanticModel { return { name: 'm', @@ -631,63 +625,62 @@ function mnModel(): SemanticModel { ], relationships: [{ name: 'order-customer', - source: {entity: 'orders', columns: []}, - destination: {entity: 'customer', columns: []}, + source: {entity: 'orders', columns: ['j_orderkey']}, + destination: {entity: 'customer', columns: ['j_custkey']}, description: 'which customers an order reached', - association: { - dataSource: 'p.d.order_customer', - keys: ['id'], - sourceColumns: ['j_orderkey'], - destinationColumns: ['j_custkey'], - fields: [{name: 'discount', expression: 'j.discount', type: 'Decimal'}], - }, + through: 'p.d.order_customer', + keys: ['id'], + fields: [{name: 'discount', expression: 'j.discount', type: 'Decimal'}], }], }; } -// The sole semantic-association entry of a generated model. -function associationEntry(model: SemanticModel) { +// The sole semantic-relationship entry of a generated model. +function relationshipEntry(model: SemanticModel) { const {entries} = generateCatalogResources(model, OPTS); - const found = - entries.filter(e => e.entryType.endsWith('/entryTypes/semantic-association')); + const found = entries.filter( + e => e.entryType.endsWith('/entryTypes/semantic-relationship')); expect(found.length).toBe(1); return found[0]; } -// A many-to-many relationship has no built-in Knowledge Catalog type: it is not -// a schema-join (a junction is two joins) and Dataplex has no custom entry LINK -// types. It publishes as an entry of the custom `semantic-association` type -// instead, the same mechanism actions use. See kc_associations.ts. -describe('a many-to-many relationship becomes a semantic-association entry', () => { +// A relationship has no built-in Knowledge Catalog type of its own: schema-join +// is an entry LINK, which holds one column pair and no name, and Dataplex has +// no custom entry LINK types. Every relationship therefore publishes as an entry +// of the custom `semantic-relationship` type, the same mechanism actions use. +// See kc_relationships.ts. +describe('a relationship becomes a semantic-relationship entry', () => { + const ASPECT = 'dest-proj.global.semantic-relationship'; + test('the entry is typed, named and parented to the model anchor', () => { - const entry = associationEntry(mnModel()); - expect(entry.name!.endsWith('/entries/m.associations.order-customer')) + const entry = relationshipEntry(mnModel()); + expect(entry.name!.endsWith('/entries/m.relationships.order-customer')) .toBe(true); // The custom type lives in the DESTINATION project (kcmd init creates it // there), not under dataplex-types with the built-in types. expect(entry.entryType) .toBe( - 'projects/dest-proj/locations/global/entryTypes/semantic-association'); + 'projects/dest-proj/locations/global/entryTypes/semantic-relationship'); expect(entry.parentEntry!.endsWith('/entries/m')).toBe(true); - // The authored name rides the entry source verbatim, so unlike a - // schema-join relationship it is not normalized on the way back. + // The authored name rides the entry source verbatim, so unlike the + // schema-join link's id it is not normalized on the way back. expect(entry.entrySource!.displayName).toBe('order-customer'); expect(entry.entrySource!.description) .toBe('which customers an order reached'); }); - test('the aspect carries both endpoints, the junction and its columns', () => { - const entry = associationEntry(mnModel()); - const aspect = entry.aspects!['dest-proj.global.semantic-association']; + test('the aspect carries both endpoints, the table and its columns', () => { + const entry = relationshipEntry(mnModel()); + const aspect = entry.aspects![ASPECT]; expect(aspect.aspectType) .toBe( - 'projects/dest-proj/locations/global/aspectTypes/semantic-association'); + 'projects/dest-proj/locations/global/aspectTypes/semantic-relationship'); const data = aspect.data!; expect(data.fromEntity).toBe('orders'); expect(data.toEntity).toBe('customer'); - // The junction is addressed the way an entity's backing table is: the - // BigQuery linked-resource URI, not the dotted form. - expect(data.junction) + // The table the edge runs through is addressed the way an entity's backing + // table is: the BigQuery linked-resource URI, not the dotted form. + expect(data.through) .toBe( '//bigquery.googleapis.com/projects/p/datasets/d/tables/order_customer'); expect(data.keys).toEqual(['id']); @@ -695,59 +688,51 @@ describe('a many-to-many relationship becomes a semantic-association entry', () expect(data.toColumns).toEqual(['j_custkey']); }); - test('edge properties ride the association aspect, not a schema aspect', () => { + test('edge properties ride the relationship aspect, not a schema aspect', () => { // The entry's type is custom, so the built-in `schema` aspect type is not // available to it (a pull derives the aspect base from the entry type's // project). The edge's own fields therefore live on this aspect. - const entry = associationEntry(mnModel()); - expect(Object.keys(entry.aspects!)).toEqual([ - 'dest-proj.global.semantic-association' - ]); - const data = entry.aspects!['dest-proj.global.semantic-association'].data!; - expect(data.fields).toEqual([ + const entry = relationshipEntry(mnModel()); + expect(Object.keys(entry.aspects!)).toEqual([ASPECT]); + expect(entry.aspects![ASPECT].data!.fields).toEqual([ {name: 'discount', dataType: 'Decimal', expression: 'j.discount'}, ]); }); test('an untyped edge property is published as Opaque', () => { const model = mnModel(); - delete model.relationships[0].association!.fields![0].type; - const data = associationEntry(model) - .aspects!['dest-proj.global.semantic-association'] - .data!; + delete model.relationships[0].fields![0].type; + const data = relationshipEntry(model).aspects![ASPECT].data!; expect(data.fields[0].dataType).toBe('Opaque'); }); - test('ai_context.instructions ride the association aspect too', () => { + test('ai_context.instructions ride the relationship aspect too', () => { const model = mnModel(); model.relationships[0].aiContext = {instructions: 'one row per pairing'}; - const data = associationEntry(model) - .aspects!['dest-proj.global.semantic-association'] - .data!; - expect(data.instructions).toBe('one row per pairing'); + const entry = relationshipEntry(model); + expect(entry.aspects![ASPECT].data!.instructions).toBe('one row per pairing'); // Not the built-in guidelines aspect, which this entry type cannot require. - expect(Object.keys(associationEntry(model).aspects!)).toEqual([ - 'dest-proj.global.semantic-association' - ]); + expect(Object.keys(entry.aspects!)).toEqual([ASPECT]); }); - test('the associations prefix is owned, so a dropped edge is reconciled', () => { - const {ownedPrefixes} = generateCatalogResources(mnModel(), OPTS); - expect(ownedPrefixes).toContain('m.associations.'); - }); + test('the relationships prefix is owned, so a dropped edge is reconciled', + () => { + const {ownedPrefixes} = generateCatalogResources(mnModel(), OPTS); + expect(ownedPrefixes).toContain('m.relationships.'); + }); - test('a model of only foreign-key edges emits no association entry', () => { - const model = mnModel(); - delete model.relationships[0].association; - model.relationships[0].source.columns = ['custkey']; - model.relationships[0].destination.columns = ['c_key']; - const {entries, ownedPrefixes} = generateCatalogResources(model, OPTS); - expect(entries.some( - e => e.entryType.endsWith('/entryTypes/semantic-association'))) - .toBe(false); - // The prefix is still owned, so an edge deleted from the model has its - // entry removed on the next push. - expect(ownedPrefixes).toContain('m.associations.'); + test('a foreign-key edge gets an entry too, with no through table', () => { + // The entry is the fidelity record for EVERY relationship; the schema-join + // link is the graph-shaped projection a foreign-key edge also gets. + const {entries, entryLinks} = + generateCatalogResources(directFkModel(), OPTS); + const entry = entries.find( + e => e.entryType.endsWith('/entryTypes/semantic-relationship'))!; + const data = entry.aspects![ASPECT].data!; + expect('through' in data).toBe(false); + expect(data.fromColumns).toEqual(['custkey']); + expect(data.toColumns).toEqual(['c_key']); + expect(entryLinks.length).toBe(1); }); test('an edge to an unpublished entity is skipped and warned', () => { @@ -755,21 +740,19 @@ describe('a many-to-many relationship becomes a semantic-association entry', () model.relationships[0].destination.entity = 'ghost'; const {entries, warnings} = generateCatalogResources(model, OPTS); expect(entries.some( - e => e.entryType.endsWith('/entryTypes/semantic-association'))) + e => e.entryType.endsWith('/entryTypes/semantic-relationship'))) .toBe(false); expect(warnings.some( w => w.includes('order-customer') && w.includes('ghost'))) .toBe(true); }); - test('a logical-only junction omits the table rather than storing a blank', + test('an unbound through table is omitted rather than stored as a blank', () => { const model = mnModel(); - model.relationships[0].association!.dataSource = ''; - const data = associationEntry(model) - .aspects!['dest-proj.global.semantic-association'] - .data!; - expect('junction' in data).toBe(false); + model.relationships[0].through = ''; + const data = relationshipEntry(model).aspects![ASPECT].data!; + expect('through' in data).toBe(false); // The logical shape survives: the edge still says what it pairs. expect(data.fromColumns).toEqual(['j_orderkey']); }); @@ -811,8 +794,10 @@ describe('a purely logical model (no physical binding) emits cleanly', () => { test('every entity is published (anchor + 2 entities), no warnings', () => { const {entries, warnings} = generateCatalogResources(logicalModel(), OPTS); const kinds = entries.map(e => e.entryType.replace(/.*\//, '')); - expect(kinds).toEqual( - ['semantic-model', 'semantic-entity', 'semantic-entity']); + expect(kinds).toEqual([ + 'semantic-model', 'semantic-entity', 'semantic-entity', + 'semantic-relationship' + ]); expect(warnings).toEqual([]); }); diff --git a/toolbox/mdcode/tests/libts/semantic/loader.test.ts b/toolbox/mdcode/tests/libts/semantic/loader.test.ts index 9cb50fb7..fbd0fa5a 100644 --- a/toolbox/mdcode/tests/libts/semantic/loader.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/loader.test.ts @@ -290,7 +290,7 @@ describe('relationships map onto the direct-FK IR convention', () => { }); }); -describe('a many-to-many relationship is bound by its association', () => { +describe('a relationship that runs through a table of its own', () => { // Two datasets plus one relationship, so each case below only has to supply // the relationship body under test. const school = (relationship: object, version = '0.2.0.dev0/google') => @@ -306,82 +306,87 @@ describe('a many-to-many relationship is bound by its association', () => { }); const full = { - association: { - source: 'enrollment', - keys: ['enrollment_id'], - from_columns: ['student_id'], - to_columns: ['course_id'], - fields: [{ name: 'grade', expression: 'enrollment.grade' }], - }, + through: 'enrollment', + keys: ['enrollment_id'], + from_columns: ['student_id'], + to_columns: ['course_id'], + fields: [{ name: 'grade', expression: 'enrollment.grade' }], }; - test('the junction table, its key, and the endpoint columns land on the IR', () => { - const rel = school(full).models[0].relationships[0]; - expect(rel.association).toEqual({ - // `source` is qualified like any other table binding. - dataSource: 'enrollment', - keys: ['enrollment_id'], - // from/to on the association are columns ON THE JUNCTION, so they map to - // the IR's sourceColumns/destinationColumns rather than to the endpoints. - sourceColumns: ['student_id'], - destinationColumns: ['course_id'], - fields: [{ name: 'grade', expression: 'enrollment.grade' }], - }); - }); - - test('neither endpoint carries join columns: no foreign key exists', () => { + test('the table, its key, the endpoint columns, and its fields land on the IR', () => { const rel = school(full).models[0].relationships[0]; - expect(rel.source).toEqual({ entity: 'students', columns: [] }); - expect(rel.destination).toEqual({ entity: 'courses', columns: [] }); + // `through` is qualified like any other table binding. + expect(rel.through).toBe('enrollment'); + expect(rel.keys).toEqual(['enrollment_id']); + // from/to columns are the same authored keys a foreign-key edge uses; what + // `through` changes is that they sit on THAT table, not on the endpoints. + expect(rel.source).toEqual({ entity: 'students', columns: ['student_id'] }); + expect(rel.destination).toEqual({ entity: 'courses', columns: ['course_id'] }); + expect(rel.fields).toEqual([{ name: 'grade', expression: 'enrollment.grade' }]); }); test('an omitted key defaults to the two column lists, deduplicated', () => { const rel = school({ - association: { - source: 'enrollment', - from_columns: ['student_id'], - to_columns: ['course_id'], - }, + through: 'enrollment', + from_columns: ['student_id'], + to_columns: ['course_id'], }).models[0].relationships[0]; - expect(rel.association!.keys).toEqual(['student_id', 'course_id']); + expect(rel.keys).toEqual(['student_id', 'course_id']); }); test('a column shared by both endpoints appears once in the default key', () => { const rel = school({ - association: { source: 'j', from_columns: ['a', 'b'], to_columns: ['b', 'c'] }, + through: 'j', from_columns: ['a', 'b'], to_columns: ['b', 'c'], }).models[0].relationships[0]; - expect(rel.association!.keys).toEqual(['a', 'b', 'c']); + expect(rel.keys).toEqual(['a', 'b', 'c']); }); test('the endpoint column lists need not be the same length', () => { // They reference two different entities' keys, so a composite key on one - // side and a single column on the other is a valid junction -- unlike a - // direct foreign key, whose two lists pair up positionally. + // side and a single column on the other is valid -- unlike a direct foreign + // key, whose two lists pair up positionally. const rel = school({ - association: { source: 'j', from_columns: ['a', 'b'], to_columns: ['c'] }, + through: 'j', from_columns: ['a', 'b'], to_columns: ['c'], }).models[0].relationships[0]; - expect(rel.association!.sourceColumns).toEqual(['a', 'b']); - expect(rel.association!.destinationColumns).toEqual(['c']); + expect(rel.source.columns).toEqual(['a', 'b']); + expect(rel.destination.columns).toEqual(['c']); }); - test('an association alongside the relationship\'s own join columns is a hard error', () => { - // The two are alternative bindings. Accepting both would leave it undefined - // which one the graph is built from. - expect(() => school({ ...full, from_columns: ['id'], to_columns: ['id'] })) - .toThrow(/must be removed/); + test('a through table with only one endpoint column list is a hard error', () => { + expect(() => school({ + through: 'enrollment', from_columns: ['student_id'], + })).toThrow(); + }); + + test('a through table with no endpoint columns at all is a hard error', () => { + // A logical edge may omit both lists; one running through a table may not. + // Without them nothing says which pairs that table holds. + expect(() => school({ through: 'enrollment' })) + .toThrow(/must give from_columns and to_columns/); }); - test('an association with only one endpoint column list is a hard error', () => { + test('a key without a through table is a hard error', () => { + // A foreign-key edge is carried by its source entity's table, so it has no + // table of its own for a key to be on. expect(() => school({ - association: { source: 'enrollment', from_columns: ['student_id'] }, - })).toThrow(); + keys: ['enrollment_id'], from_columns: ['id'], to_columns: ['id'], + })).toThrow(/needs a 'through' table/); + }); + + test('fields without a through table are a hard error', () => { + // Same reason: an edge with no table of its own has nowhere to hold + // properties of the pairing. + expect(() => school({ + fields: [{ name: 'grade', expression: 'x.grade' }], + from_columns: ['id'], to_columns: ['id'], + })).toThrow(/needs a 'through' table/); }); - test('vanilla Ossie rejects the association key', () => { - // Many-to-many is a native extension of the extended profile; vanilla - // Ossie has no junction-table syntax and no carrier for one, so the key is - // unknown rather than silently dropped. - expect(() => school(full, '0.2.0.dev0')).toThrow(/association/); + test('vanilla Ossie rejects the through key', () => { + // An edge through a table of its own is a native extension of the extended + // profile; vanilla Ossie knows only the direct foreign key and has no + // carrier for the rest, so the key is unknown rather than silently dropped. + expect(() => school(full, '0.2.0.dev0')).toThrow(/through/); }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts index 474daa9d..3b2d13cd 100644 --- a/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_converter.test.ts @@ -188,18 +188,14 @@ describe('expression + datatype + dimension mapping', () => { describe('many-to-many relationships', () => { - test('an association round-trips whole', () => { + test('an edge through a table of its own round-trips whole', () => { const rel: Relationship = { name: 'enrollment', - source: {entity: 'student', columns: []}, - destination: {entity: 'course', columns: []}, - association: { - dataSource: 'p.d.enrollment', - keys: ['student_id', 'course_id'], - sourceColumns: ['student_id'], - destinationColumns: ['course_id'], - fields: [{name: 'grade', expression: 'grade', type: 'String'}], - }, + source: {entity: 'student', columns: ['student_id']}, + destination: {entity: 'course', columns: ['course_id']}, + through: 'p.d.enrollment', + keys: ['student_id', 'course_id'], + fields: [{name: 'grade', expression: 'grade', type: 'String'}], }; const model: SemanticModel = { name: 'school', @@ -211,26 +207,24 @@ describe('many-to-many relationships', () => { metrics: [], }; const {yaml: text, warnings} = serializeModel(model); - expect(warnings.some(w => /association/i.test(w))).toBe(false); + expect(warnings.some(w => /through/i.test(w))).toBe(false); - // The junction detail is a native key now, so it survives serialization - // instead of collapsing to a direct-FK view. + // `through` and what it brings are native keys now, so they survive + // serialization instead of collapsing to a direct-FK view. const relDoc = yaml.parse(text).semantic_model[0].relationships[0]; expect(relDoc.from).toBe('student'); expect(relDoc.to).toBe('course'); - // A many-to-many edge carries no join columns of its own; the columns that - // bind it are on the junction table. - expect(relDoc.from_columns).toBeUndefined(); - expect(relDoc.to_columns).toBeUndefined(); - expect(relDoc.association.source).toBe('p.d.enrollment'); - expect(relDoc.association.keys).toEqual(['student_id', 'course_id']); - expect(relDoc.association.from_columns).toEqual(['student_id']); - expect(relDoc.association.to_columns).toEqual(['course_id']); - expect(relDoc.association.fields[0].name).toBe('grade'); + expect(relDoc.through).toBe('p.d.enrollment'); + expect(relDoc.keys).toEqual(['student_id', 'course_id']); + // The join columns are the same keys a foreign-key edge uses; `through` + // says they are on that table rather than on either endpoint. + expect(relDoc.from_columns).toEqual(['student_id']); + expect(relDoc.to_columns).toEqual(['course_id']); + expect(relDoc.fields[0].name).toBe('grade'); // And it reloads into the same IR. const reloaded = fromDocument(yaml.parse(text)).models[0]; - expect(reloaded.relationships[0].association).toEqual(rel.association!); + expect(reloaded.relationships[0]).toEqual(rel); }); }); diff --git a/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts b/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts index a38b6b29..5c42e2de 100644 --- a/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/osi_schema.test.ts @@ -168,26 +168,21 @@ function onlyActionsExtension(errors: typeof validate.errors): boolean { /\/semantic_model\/\d+$/.test(e.instancePath)); } -// A many-to-many `association` block is the other deliberate SUPERSET of -// released Apache OSI. The released schema knows only the direct foreign-key -// edge, so a junction-backed relationship trips it twice: `association` is an -// additional property, and the `from_columns`/`to_columns` it required are -// absent -- correctly, because a many-to-many edge has none of its own. We -// tolerate EXACTLY those three errors on a /relationships/ path and nothing -// else. When upstream OSI adopts a junction-table syntax, re-vendoring the -// schema makes this pass with no special-casing. -function isAssociationSuperset(e: SchemaError): boolean { - const missingOk = new Set(['from_columns', 'to_columns']); - if (!/\/relationships\/\d+$/.test(e.instancePath)) return false; - if (e.keyword === 'required') { - return missingOk.has( - (e.params as {missingProperty?: string}).missingProperty ?? ''); - } - if (e.keyword === 'additionalProperties') { - return (e.params as {additionalProperty?: string}).additionalProperty === - 'association'; - } - return false; +// An edge that runs THROUGH a table of its own is the other deliberate SUPERSET +// of released Apache OSI. The released schema knows only the direct foreign-key +// edge -- one carried by a column on the source entity's own table -- so it has +// no keyword for the table an edge runs through, for that table's key, or for +// the properties of the pairing it holds. A relationship carrying them trips +// `additionalProperties: false` once per keyword. We tolerate EXACTLY those +// three extra properties on a /relationships/ path and nothing else. When +// upstream OSI adopts a through-table syntax, re-vendoring the schema makes this +// pass with no special-casing. +function isThroughSuperset(e: SchemaError): boolean { + const extraOk = new Set(['through', 'keys', 'fields']); + return e.keyword === 'additionalProperties' && + /\/relationships\/\d+$/.test(e.instancePath) && + extraOk.has( + (e.params as {additionalProperty?: string}).additionalProperty ?? ''); } describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => { @@ -203,14 +198,13 @@ describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => if (!ok) { // A .pull.golden.yaml from an expression-free push is a known #290 gap // when its ONLY failures are missing `expression`; anything else is a - // real regression and still fails. A junction-backed fixture's pull - // faithfully reproduces the association superset too, so that one - // tolerates both. + // real regression and still fails. A through-edge fixture's pull + // faithfully reproduces that superset too, so that one tolerates both. if (rel.endsWith('.pull.golden.yaml') && onlyTolerated( validate.errors, rel.startsWith('school_manytomany') ? - [isMissingExpression, isAssociationSuperset] : + [isMissingExpression, isThroughSuperset] : [isMissingExpression])) { return; } @@ -238,11 +232,11 @@ describe('fixtures are valid Apache OSI (osi-schema.json, Draft 2020-12)', () => onlyActionsExtension(validate.errors)) { return; } - // A junction-backed relationship is a deliberate superset too; tolerate - // exactly its three errors, and only on the fixture that carries one - // (and the goldens generated from it). + // A relationship running through a table of its own is a deliberate + // superset too; tolerate exactly its three extra keywords, and only on + // the fixture that carries one (and the goldens generated from it). if (rel.startsWith('school_manytomany') && - onlyTolerated(validate.errors, [isAssociationSuperset])) { + onlyTolerated(validate.errors, [isThroughSuperset])) { return; } const details = (validate.errors ?? []) diff --git a/toolbox/mdcode/tests/libts/semantic/resolve_profiles.test.ts b/toolbox/mdcode/tests/libts/semantic/resolve_profiles.test.ts index 0424d6e9..bca8d820 100644 --- a/toolbox/mdcode/tests/libts/semantic/resolve_profiles.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/resolve_profiles.test.ts @@ -263,6 +263,21 @@ describe('pruneUnavailable drops what a binding cannot answer', () => { expect(report.droppedRelationships[0].name).toBe('PlacedBy'); }); + test('a through-edge is not dropped by an unbound field of the same name', + () => { + // The edge's join columns are on the table it runs THROUGH, not on + // either endpoint, so a profile cannot unbind them -- and a same-named + // field on the endpoint must not be mistaken for one. + const m = irModel(); + const order = m.entities.find(e => e.name === 'Order')!; + delete order.fields.find(f => f.name === 'customerKey')!.expression; + m.relationships[0].through = 'p.d.order_customer'; + m.relationships[0].keys = ['id']; + const {model, report} = pruneUnavailable(m, 'operational'); + expect(relNames(model)).toEqual(['PlacedBy']); + expect(report.droppedRelationships).toEqual([]); + }); + test('the input is never mutated', () => { const m = irModel(); const before = JSON.stringify(m); diff --git a/toolbox/mdcode/tests/libts/semantic/spanner.test.ts b/toolbox/mdcode/tests/libts/semantic/spanner.test.ts index 3aff9896..03a335ff 100644 --- a/toolbox/mdcode/tests/libts/semantic/spanner.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/spanner.test.ts @@ -6,8 +6,8 @@ // showing the exact generated DDL and warnings. Prefer adding a fixture + // golden there. // -// This file holds only what a loader fixture CANNOT express: an M:N association -// edge (the open format has no association-table syntax, so its IR is +// This file holds only what a loader fixture CANNOT express: a through-edge +// named entirely with reserved words (the loader rejects those, so its IR is // hand-built and checked against a committed golden), and degenerate/negative // inputs and pure GenerateOptions behavior (graph naming, bare table mapping). @@ -30,15 +30,15 @@ function loadFixture(fixture: string): SemanticModel { return models[0]; } -describe('M:N association edge', () => { +describe('many-to-many edge', () => { // Loaded from `school_manytomany.yaml`, the same document the BigQuery suite // renders, so one authored many-to-many model is shown deploying to either // store. The expected DDL is a committed golden // (`school_manytomany.spanner.golden.sql`), the Spanner counterpart to the - // BigQuery association golden, so the two shapes are reviewable side by side. + // BigQuery golden, so the two shapes are reviewable side by side. const SCHOOL = loadFixture('school_manytomany.yaml'); - test('the association graph matches its committed golden DDL', () => { + test('the many-to-many graph matches its committed golden DDL', () => { const {ddl} = generateSpannerPropertyGraph(SCHOOL); const golden = path.join(FIXTURES, 'school_manytomany.spanner.golden.sql'); if (process.env.UPDATE_GOLDENS) { @@ -51,7 +51,7 @@ describe('M:N association edge', () => { test( 'an edge property carries no OPTIONS (Spanner has no per-element options)', () => { - // The junction's `grade` field has a description; on BigQuery that + // The edge's own `grade` field has a description; on BigQuery that // becomes an OPTIONS clause, on Spanner it is dropped. const {ddl} = generateSpannerPropertyGraph(SCHOOL); expect(ddl).toContain('grade'); @@ -289,11 +289,11 @@ describe('degenerate inputs', () => { }); }); -describe('reserved-word names in an M:N association edge are quoted', () => { - // The open format has no association-table syntax, so this hand-built IR is - // the only path that exercises renderAssociationEdge's identifier quoting on - // the Spanner leg: the edge alias, KEY, SOURCE KEY / DESTINATION KEY columns, - // and both REFERENCES labels, each named with a GoogleSQL reserved keyword. +describe('reserved-word names in a through-edge are quoted', () => { + // The loader rejects reserved-word names, so this hand-built IR is the only + // path that exercises renderThroughEdge's identifier quoting on the Spanner + // leg: the edge alias, KEY, SOURCE KEY / DESTINATION KEY columns, and both + // REFERENCES labels, each named with a GoogleSQL reserved keyword. // Table names stay bare (Spanner graphs live in one database). const RW_ASSOC: SemanticModel = { name: 'rw_assoc', @@ -315,13 +315,8 @@ describe('reserved-word names in an M:N association edge are quoted', () => { name: 'from', source: {entity: 'Order', columns: ['order']}, destination: {entity: 'Group', columns: ['id']}, - association: { - dataSource: 'proj.ds.order_group', - keys: ['order'], - sourceColumns: ['order'], - destinationColumns: ['id'], - fields: [], - }, + through: 'proj.ds.order_group', + keys: ['order'], }], metrics: [], }; diff --git a/toolbox/mdcode/tests/libts/semantic/transpile.test.ts b/toolbox/mdcode/tests/libts/semantic/transpile.test.ts index bba38254..b05a7a5d 100644 --- a/toolbox/mdcode/tests/libts/semantic/transpile.test.ts +++ b/toolbox/mdcode/tests/libts/semantic/transpile.test.ts @@ -195,8 +195,8 @@ describe('transpileModel', () => { expect(input).toEqual(snapshot); }); - test('transpiles association (junction) edge-property fields', async () => { - const withAssoc: SemanticModel = { + test('transpiles a through-edge\'s own property fields', async () => { + const withThrough: SemanticModel = { name: 'm', entities: [ {name: 'student', dataSource: 'p.d.student', keys: ['sid'], fields: []}, @@ -206,24 +206,20 @@ describe('transpileModel', () => { name: 'enrollment', source: {entity: 'student', columns: ['sid']}, destination: {entity: 'course', columns: ['cid']}, - association: { - dataSource: 'p.d.enrollment', - keys: ['eid'], - sourceColumns: ['sid'], - destinationColumns: ['cid'], - fields: [{ - name: 'grade', - importedExpression: 'IFF(g>0,g,0)', - importedDialect: 'SNOWFLAKE' - }], - }, + through: 'p.d.enrollment', + keys: ['eid'], + fields: [{ + name: 'grade', + importedExpression: 'IFF(g>0,g,0)', + importedDialect: 'SNOWFLAKE' + }], }], metrics: [], }; const {transpiler} = fakeTranspiler(() => 'IF(g > 0, g, 0)'); - const {model, warnings} = await transpileModel(withAssoc, {transpiler}); + const {model, warnings} = await transpileModel(withThrough, {transpiler}); - expect(model.relationships[0].association!.fields![0].expression) + expect(model.relationships[0].fields![0].expression) .toBe('IF(g > 0, g, 0)'); expect(warnings.some( w => w.includes(`relationship 'enrollment' field 'grade'`) && diff --git a/toolbox/mdcode/tests/tool/init_semantic_model.test.ts b/toolbox/mdcode/tests/tool/init_semantic_model.test.ts index b58e1ad3..0337e8c5 100644 --- a/toolbox/mdcode/tests/tool/init_semantic_model.test.ts +++ b/toolbox/mdcode/tests/tool/init_semantic_model.test.ts @@ -4,8 +4,8 @@ // Both are created at init -- not on push -- so a semantic-model push writes // only entries, matching how the standard layout operates (its push creates // entries, never the entry group). The types kcmd creates rather than -// references are declared in kc_custom_types.ts, which today holds the one -// action pair and the many-to-many association pair. The tests below assert +// references are declared in kc_custom_types.ts, which today holds the action +// pair and the relationship pair. The tests below assert // over CUSTOM_TYPES rather than naming those two, so adding a third type does // not need them rewritten. They spy on the catalog client so no network call is // made and run init