diff --git a/converters/README.md b/converters/README.md index 014a7350..59a50035 100644 --- a/converters/README.md +++ b/converters/README.md @@ -232,7 +232,7 @@ A converter should map `ai_context` when the target vendor supports equivalent c 1. **Validate input**: Use the [Ossie JSON Schema](../core-spec/ossie-schema.json) and the [validation script](../validation/validate.py) to ensure the source Ossie model is valid before conversion. -2. **Parse the Ossie model**: Load the YAML file and iterate over the top-level `semantic_model` entries. +2. **Parse the Ossie model**: Load the JSON or YAML document as one model. 3. **Map datasets**: For each dataset, translate the `name`, `source`, `primary_key`, `unique_keys`, and `fields` to the vendor's format. Parse the `source` string (typically `database.schema.table`) into the vendor's catalog structure. diff --git a/core-spec/ossie-schema.json b/core-spec/ossie-schema.json index 5810af84..decf6e42 100644 --- a/core-spec/ossie-schema.json +++ b/core-spec/ossie-schema.json @@ -2,7 +2,7 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://github.com/apache/ossie/core-spec/ossie-schema.json", "title": "Apache Ossie Core Metadata Specification", - "description": "JSON Schema for validating Apache Ossie semantic model definitions", + "description": "JSON Schema for validating a single Apache Ossie semantic model document", "type": "object", "properties": { "version": { @@ -10,15 +10,29 @@ "const": "0.2.0.dev0", "description": "Apache Ossie specification version" }, - "semantic_model": { - "type": "array", - "description": "Collection of semantic model definitions", - "items": { - "$ref": "#/$defs/SemanticModel" - } + "name": { + "$ref": "#/$defs/SemanticModel/properties/name" + }, + "description": { + "$ref": "#/$defs/SemanticModel/properties/description" + }, + "ai_context": { + "$ref": "#/$defs/SemanticModel/properties/ai_context" + }, + "datasets": { + "$ref": "#/$defs/SemanticModel/properties/datasets" + }, + "relationships": { + "$ref": "#/$defs/SemanticModel/properties/relationships" + }, + "metrics": { + "$ref": "#/$defs/SemanticModel/properties/metrics" + }, + "custom_extensions": { + "$ref": "#/$defs/SemanticModel/properties/custom_extensions" } }, - "required": ["version", "semantic_model"], + "required": ["version", "name", "datasets"], "additionalProperties": false, "$defs": { "Dialect": { diff --git a/core-spec/spec.md b/core-spec/spec.md index 6cf9a11e..86bbc636 100644 --- a/core-spec/spec.md +++ b/core-spec/spec.md @@ -84,12 +84,17 @@ ontology specification's built-in value types; `Time`, `DateTimeTz`, and ## Semantic Model -The top-level container that represents a complete semantic model, including datasets, relationships, and metrics. +Each JSON or YAML document represents exactly one semantic model. + +A standalone document must contain `version`, `name`, and a non-empty `datasets` +array. For bulk exchange, use separate model documents. This specification does +not define a bundle format or cross-model references. ### Schema | Field | Type | Required | Description | |-------|------|----------|-------------| +| `version` | string | Yes | Apache Ossie specification version (`0.2.0.dev0`) | | `name` | string | Yes | Unique identifier for the semantic model | | `description` | string | No | Human-readable description | | `ai_context` | string/object | No | Additional context for AI tools (e.g., custom instructions) | @@ -101,21 +106,54 @@ The top-level container that represents a complete semantic model, including dat ### Example ```yaml -semantic_model: - - name: sales_analytics - description: Sales and customer analytics model - ai_context: - instructions: "Use this model for sales analysis and customer insights" - datasets: - - name: orders - source: sales.public.orders - relationships: [] - metrics: [] - custom_extensions: - - vendor_name: DBT - data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' +version: 0.2.0.dev0 +name: sales_analytics +description: Sales and customer analytics model +ai_context: + instructions: "Use this model for sales analysis and customer insights" +datasets: + - name: orders + source: sales.public.orders +relationships: [] +metrics: [] +custom_extensions: + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' ``` +The same document structure in JSON: + +```json +{ + "version": "0.2.0.dev0", + "name": "sales_analytics", + "datasets": [ + {"name": "orders", "source": "sales.public.orders"} + ] +} +``` + +### Migrating earlier document shapes + +This is a breaking change in the unreleased `0.2.0.dev0` specification. Earlier +releases and earlier development snapshots use a `semantic_model` array. The +current schema accepts only the flat document shape; it does not accept the array +or an object-valued wrapper. + +To migrate a document containing one model, move that model's properties to the +root and remove `semantic_model`. Use `version: 0.2.0.dev0` for the migrated +document. Remove any root-level `dialects` and `vendors` declarations; preserve +per-expression dialects and vendor information in `custom_extensions`. For +multiple models, create one document per model and validate each result. An empty +model array cannot produce a valid model document. Preserve model contents and +custom extensions; never silently select only the first model or overwrite a file +when splitting a document. + +The reusable `$defs/SemanticModel` schema still describes model contents without +standalone document metadata. In particular, an ontology map continues to embed +those contents under its `semantic_model` property. This standalone document +change does not rename or flatten that ontology property. + --- ## Datasets @@ -507,101 +545,100 @@ Here's a complete semantic model example showing all components working together ```yaml version: 0.2.0.dev0 -semantic_model: - - name: ecommerce_analytics - description: E-commerce sales and customer analytics - ai_context: - instructions: "Use this model for analyzing sales trends, customer behavior, and product performance" - - datasets: - - name: orders - source: sales.public.orders - primary_key: [order_id] - description: Customer orders - fields: - - name: order_id - expression: - dialects: - - dialect: ANSI_SQL - expression: order_id - description: Order identifier - - - name: customer_id - expression: - dialects: - - dialect: ANSI_SQL - expression: customer_id - description: Customer identifier - - - name: order_date - expression: - dialects: - - dialect: ANSI_SQL - expression: order_date - datatype: Date - dimension: - is_time: true - description: Order date - - - name: amount - expression: - dialects: - - dialect: ANSI_SQL - expression: amount - description: Order amount - - - name: customers - source: sales.public.customers - primary_key: [id] - description: Customer information - fields: - - name: id - expression: - dialects: - - dialect: ANSI_SQL - expression: id - description: Customer identifier - - - name: email - expression: - dialects: - - dialect: ANSI_SQL - expression: email - description: Customer email - - relationships: - - name: orders_to_customers - from: orders - to: customers - from_columns: [customer_id] - to_columns: [id] - - metrics: - - name: total_revenue +name: ecommerce_analytics +description: E-commerce sales and customer analytics +ai_context: + instructions: "Use this model for analyzing sales trends, customer behavior, and product performance" + +datasets: + - name: orders + source: sales.public.orders + primary_key: [order_id] + description: Customer orders + fields: + - name: order_id expression: dialects: - dialect: ANSI_SQL - expression: SUM(orders.amount) - description: Total revenue from all orders - ai_context: - synonyms: - - "total sales" - - "revenue" - - - name: customer_count + expression: order_id + description: Order identifier + + - name: customer_id expression: dialects: - dialect: ANSI_SQL - expression: COUNT(DISTINCT customers.id) - description: Total number of customers - ai_context: - synonyms: - - "total customers" - - "customer base" + expression: customer_id + description: Customer identifier - custom_extensions: - - vendor_name: SNOWFLAKE - data: '{"warehouse": "ANALYTICS_WH"}' + - name: order_date + expression: + dialects: + - dialect: ANSI_SQL + expression: order_date + datatype: Date + dimension: + is_time: true + description: Order date + + - name: amount + expression: + dialects: + - dialect: ANSI_SQL + expression: amount + description: Order amount + + - name: customers + source: sales.public.customers + primary_key: [id] + description: Customer information + fields: + - name: id + expression: + dialects: + - dialect: ANSI_SQL + expression: id + description: Customer identifier + + - name: email + expression: + dialects: + - dialect: ANSI_SQL + expression: email + description: Customer email + +relationships: + - name: orders_to_customers + from: orders + to: customers + from_columns: [customer_id] + to_columns: [id] + +metrics: + - name: total_revenue + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(orders.amount) + description: Total revenue from all orders + ai_context: + synonyms: + - "total sales" + - "revenue" + + - name: customer_count + expression: + dialects: + - dialect: ANSI_SQL + expression: COUNT(DISTINCT customers.id) + description: Total number of customers + ai_context: + synonyms: + - "total customers" + - "customer base" + +custom_extensions: + - vendor_name: SNOWFLAKE + data: '{"warehouse": "ANALYTICS_WH"}' ``` --- @@ -643,6 +680,7 @@ ai_context: ## Version History - **0.2.0.dev0** (Unreleased): In-development next minor release. Schema is mutable; do not depend on this version in production. + - Breaking: each standalone document contains one model directly at the root; the `semantic_model` array is removed. - **0.1.1** (2025-12-11): Initial release - Core semantic model structure - Support for datasets, relationships, fields, and metrics diff --git a/core-spec/spec.yaml b/core-spec/spec.yaml index 074f69f3..d1852fb8 100644 --- a/core-spec/spec.yaml +++ b/core-spec/spec.yaml @@ -61,42 +61,44 @@ datatypes: vendor_name: string +# Standalone documents contain exactly one model directly at the root. +# No semantic_model wrapper; include version alongside the model properties. +# The sections in this reference file describe separate schema components. # Top-level semantic model definition -semantic_model: - # Required: Unique identifier for the semantic model - - name: string - - # Optional: Human-readable description of the semantic model - description: string - - # Optional: Additional context for AI tools (e.g., custom prompts, instructions) - # Can be either: - # - A free-form string, or - # - A structured object with optional keys: - # instructions: string # how AI should use this entity - # synonyms: [] # alternative names / terms - # examples: [] # sample questions or use cases - # (additionalProperties: true, so vendors may add more keys) - ai_context: {} # see AIContext in ossie-schema.json - - # Required: Collection of logical datasets (fact and dimension tables) - # See Logical Dataset section below for detailed structure - datasets: [] - - # Optional: Defines how logical datasets are connected - # See Relationships section below for detailed structure - relationships: [] - - # Optional: - # These metrics can span one or more logical datasets and use relationships - # See Metrics section below for detailed structure - metrics: [] - - # Optional: Vendor-specific attributes for extensibility - # Allows vendors to add custom metadata without breaking core compatibility - custom_extensions: - - vendor_name: string # Free-form string identifying the vendor - data: string +# Required: Unique identifier for the semantic model +name: string + +# Optional: Human-readable description of the semantic model +description: string + +# Optional: Additional context for AI tools (e.g., custom prompts, instructions) +# Can be either: +# - A free-form string, or +# - A structured object with optional keys: +# instructions: string # how AI should use this entity +# synonyms: [] # alternative names / terms +# examples: [] # sample questions or use cases +# (additionalProperties: true, so vendors may add more keys) +ai_context: {} # see AIContext in ossie-schema.json + +# Required: Collection of logical datasets (fact and dimension tables) +# See Logical Dataset section below for detailed structure +datasets: [] + +# Optional: Defines how logical datasets are connected +# See Relationships section below for detailed structure +relationships: [] + +# Optional: +# These metrics can span one or more logical datasets and use relationships +# See Metrics section below for detailed structure +metrics: [] + +# Optional: Vendor-specific attributes for extensibility +# Allows vendors to add custom metadata without breaking core compatibility +custom_extensions: + - vendor_name: string # Free-form string identifying the vendor + data: string --- # Logical Dataset Schema @@ -151,7 +153,7 @@ datasets: --- # Relationship Schema -# Defines how logical datasets or semantic models are connected +# Defines how logical datasets within this model are connected # Represents foreign key relationships (many-to-one or one-to-one) relationships: # Required: Unique identifier for the relationship diff --git a/docs/index.md b/docs/index.md index 3836092d..fb93fb14 100644 --- a/docs/index.md +++ b/docs/index.md @@ -47,7 +47,7 @@ Ossie addresses semantic fragmentation by providing: ### Specification at a Glance -The Ossie core specification (current version: **0.2.0.dev0**, latest released: **0.1.1**) defines a YAML-based format for describing semantic models. The key constructs are: +The Ossie core specification (current version: **0.2.0.dev0**, latest released: **0.1.1**) defines a JSON/YAML format with one semantic model per document. Model properties such as `name` and `datasets` sit directly at the root alongside `version`, without a `semantic_model` wrapper. Converter and Python SDK support for this development format is follow-up work. See the [migration guidance](../core-spec/spec.md#migrating-earlier-document-shapes). The key constructs are: | Construct | Description | |-----------|-------------| diff --git a/examples/tpcds_semantic_model.yaml b/examples/tpcds_semantic_model.yaml index e0bb288e..dce43439 100644 --- a/examples/tpcds_semantic_model.yaml +++ b/examples/tpcds_semantic_model.yaml @@ -22,651 +22,650 @@ version: "0.2.0.dev0" -semantic_model: - - name: tpcds_retail_model - description: TPC-DS retail semantic model for sales and customer analytics +name: tpcds_retail_model +description: TPC-DS retail semantic model for sales and customer analytics +ai_context: + instructions: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." + +datasets: + # Fact table: Store sales transactions + - name: store_sales + source: tpcds.public.store_sales + primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key + unique_keys: + - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item + description: Fact table containing all store sales transactions ai_context: - instructions: "Use this semantic model for retail analytics. It provides comprehensive sales, customer, product, and store data from the TPC-DS benchmark. The model supports time-based analysis, customer segmentation, product performance, and store operations metrics." - - datasets: - # Fact table: Store sales transactions - - name: store_sales - source: tpcds.public.store_sales - primary_key: [ss_item_sk, ss_ticket_number] # Composite primary key - unique_keys: - - [ss_item_sk, ss_ticket_number] # Composite key: item + ticket number uniquely identifies a line item - description: Fact table containing all store sales transactions + synonyms: + - "sales transactions" + - "store purchases" + - "retail sales" + - "POS data" + + fields: + - name: ss_sold_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sold_date_sk + description: Foreign key to date dimension + datatype: Integer + dimension: + is_time: false ai_context: synonyms: - - "sales transactions" - - "store purchases" - - "retail sales" - - "POS data" - - fields: - - name: ss_sold_date_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_sold_date_sk - description: Foreign key to date dimension - datatype: Integer - dimension: - is_time: false - ai_context: - synonyms: - - "sale date" - - "transaction date" - - - name: ss_item_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_item_sk - description: Foreign key to item dimension - datatype: Integer - dimension: - is_time: false - ai_context: - synonyms: - - "product" - - "item" - - - name: ss_customer_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_customer_sk - description: Foreign key to customer dimension - datatype: Integer - dimension: - is_time: false - ai_context: - synonyms: - - "customer" - - "buyer" - - - name: ss_store_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_store_sk - description: Foreign key to store dimension - datatype: Integer - dimension: - is_time: false - ai_context: - synonyms: - - "store" - - "location" - - - name: ss_quantity - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_quantity - description: Quantity of items sold - datatype: Integer - ai_context: - synonyms: - - "units sold" - - "quantity" - - - name: ss_sales_price - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_sales_price - description: Sales price per unit - datatype: Decimal - ai_context: - synonyms: - - "unit price" - - "price" - - - name: ss_ext_sales_price - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_ext_sales_price - description: Extended sales price (quantity * price) - datatype: Decimal - ai_context: - synonyms: - - "total price" - - "line total" - - - name: ss_net_profit - expression: - dialects: - - dialect: ANSI_SQL - expression: ss_net_profit - description: Net profit from the sale - datatype: Decimal - ai_context: - synonyms: - - "profit" - - "margin" - - # Dimension table: Date - - name: date_dim - source: tpcds.public.date_dim - primary_key: [d_date_sk] # Simple primary key - unique_keys: - - [d_date_sk] # Simple key: single column - description: Date dimension with calendar attributes + - "sale date" + - "transaction date" + + - name: ss_item_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_item_sk + description: Foreign key to item dimension + datatype: Integer + dimension: + is_time: false ai_context: synonyms: - - "calendar" - - "dates" - - "time periods" - - fields: - - name: d_date_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: d_date_sk - description: Surrogate key for date - datatype: Integer - dimension: - is_time: false - - - name: d_date - expression: - dialects: - - dialect: ANSI_SQL - expression: d_date - description: Actual date value - datatype: Date - dimension: {} - ai_context: - synonyms: - - "date" - - "calendar date" - - - name: d_year - expression: - dialects: - - dialect: ANSI_SQL - expression: d_year - description: Year - datatype: Integer - dimension: - is_time: true - ai_context: - synonyms: - - "year" - - # Declares temporal role via is_time without a datatype annotation. - # Both datatype and is_time are independently optional. - - name: d_quarter_name - expression: - dialects: - - dialect: ANSI_SQL - expression: d_quarter_name - description: Quarter name (e.g., 2024Q1) - dimension: - is_time: true - ai_context: - synonyms: - - "quarter" - - "fiscal quarter" - - # Declares temporal role via is_time without a datatype annotation. - # Both datatype and is_time are independently optional. - - name: d_moy - expression: - dialects: - - dialect: ANSI_SQL - expression: d_moy - description: Month of year (1-12) - dimension: - is_time: true - ai_context: - synonyms: - - "month" - - # Dimension table: Customer - - name: customer - source: tpcds.public.customer - primary_key: [c_customer_sk] # Simple primary key - unique_keys: - - [c_customer_sk] # Simple key: single column - description: Customer dimension with demographic information + - "product" + - "item" + + - name: ss_customer_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_customer_sk + description: Foreign key to customer dimension + datatype: Integer + dimension: + is_time: false ai_context: synonyms: - - "customers" - - "shoppers" - - "buyers" - - fields: - - name: c_customer_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: c_customer_sk - description: Surrogate key for customer - datatype: Integer - dimension: - is_time: false - - - name: c_customer_id - expression: - dialects: - - dialect: ANSI_SQL - expression: c_customer_id - description: Business key for customer - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "customer ID" - - "customer number" - - - name: c_first_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_first_name - description: Customer first name - datatype: String - dimension: - is_time: false - - - name: c_last_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_last_name - description: Customer last name - datatype: String - dimension: - is_time: false - - - name: customer_full_name - expression: - dialects: - - dialect: ANSI_SQL - expression: c_first_name || ' ' || c_last_name - description: Customer full name (computed field) - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "full name" - - "customer name" - - - name: c_email_address - expression: - dialects: - - dialect: ANSI_SQL - expression: c_email_address - description: Customer email address - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "email" - - "contact" - - # Dimension table: Item (Product) - - name: item - source: tpcds.public.item - primary_key: [i_item_sk] # Simple primary key - unique_keys: - - [i_item_sk] # Simple key: single column - description: Item/Product dimension with product attributes + - "customer" + - "buyer" + + - name: ss_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_store_sk + description: Foreign key to store dimension + datatype: Integer + dimension: + is_time: false ai_context: synonyms: - - "products" - - "items" - - "merchandise" - - fields: - - name: i_item_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_sk - description: Surrogate key for item - datatype: Integer - dimension: - is_time: false - - - name: i_item_id - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_id - description: Business key for item - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "item ID" - - "product ID" - - "SKU" - - - name: i_item_desc - expression: - dialects: - - dialect: ANSI_SQL - expression: i_item_desc - description: Item description - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "product description" - - "item name" - - - name: i_brand - expression: - dialects: - - dialect: ANSI_SQL - expression: i_brand - description: Brand name - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "brand" - - "manufacturer" - - - name: i_category - expression: - dialects: - - dialect: ANSI_SQL - expression: i_category - description: Item category - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "product category" - - "department" - - - name: i_current_price - expression: - dialects: - - dialect: ANSI_SQL - expression: i_current_price - description: Current price of the item - datatype: Decimal - dimension: - is_time: false - ai_context: - synonyms: - - "price" - - "list price" - - # Dimension table: Store - - name: store - source: tpcds.public.store - primary_key: [s_store_sk] # Simple primary key - unique_keys: - - [s_store_id] # Simple key: single column - description: Store dimension with location and store attributes + - "store" + - "location" + + - name: ss_quantity + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_quantity + description: Quantity of items sold + datatype: Integer ai_context: synonyms: - - "stores" - - "retail locations" - - "branches" - - fields: - - name: s_store_sk - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_sk - description: Surrogate key for store - datatype: Integer - dimension: - is_time: false - - - name: s_store_id - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_id - description: Business key for store - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "store ID" - - "store number" - - - name: s_store_name - expression: - dialects: - - dialect: ANSI_SQL - expression: s_store_name - description: Store name - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "store name" - - "location name" - - - name: s_city - expression: - dialects: - - dialect: ANSI_SQL - expression: s_city - description: City where store is located - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "city" - - "location" - - - name: s_state - expression: - dialects: - - dialect: ANSI_SQL - expression: s_state - description: State where store is located - datatype: String - dimension: - is_time: false - ai_context: - synonyms: - - "state" - - "region" - - - name: s_number_employees - expression: - dialects: - - dialect: ANSI_SQL - expression: s_number_employees - description: Number of employees at the store - datatype: Integer - ai_context: - synonyms: - - "employee count" - - "staff size" - - # Relationships between datasets - relationships: - - name: store_sales_to_date - from: store_sales - to: date_dim - from_columns: [ss_sold_date_sk] - to_columns: [d_date_sk] + - "units sold" + - "quantity" + + - name: ss_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_sales_price + description: Sales price per unit + datatype: Decimal ai_context: synonyms: - - "sales date relationship" - - "when sale occurred" - - - name: store_sales_to_customer - from: store_sales - to: customer - from_columns: [ss_customer_sk] - to_columns: [c_customer_sk] + - "unit price" + - "price" + + - name: ss_ext_sales_price + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_ext_sales_price + description: Extended sales price (quantity * price) + datatype: Decimal ai_context: synonyms: - - "customer purchase relationship" - - "who bought" - - - name: store_sales_to_item - from: store_sales - to: item - from_columns: [ss_item_sk] - to_columns: [i_item_sk] + - "total price" + - "line total" + + - name: ss_net_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: ss_net_profit + description: Net profit from the sale + datatype: Decimal ai_context: synonyms: - - "product sold relationship" - - "what was sold" - - - name: store_sales_to_store - from: store_sales - to: store - from_columns: [ss_store_sk] - to_columns: [s_store_sk] + - "profit" + - "margin" + + # Dimension table: Date + - name: date_dim + source: tpcds.public.date_dim + primary_key: [d_date_sk] # Simple primary key + unique_keys: + - [d_date_sk] # Simple key: single column + description: Date dimension with calendar attributes + ai_context: + synonyms: + - "calendar" + - "dates" + - "time periods" + + fields: + - name: d_date_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date_sk + description: Surrogate key for date + datatype: Integer + dimension: + is_time: false + + - name: d_date + expression: + dialects: + - dialect: ANSI_SQL + expression: d_date + description: Actual date value + datatype: Date + dimension: {} ai_context: synonyms: - - "store location relationship" - - "where sale occurred" + - "date" + - "calendar date" - # Semantic model-level metrics spanning multiple datasets - metrics: - - name: total_sales + - name: d_year expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - description: Total sales revenue across all transactions - datatype: Decimal + expression: d_year + description: Year + datatype: Integer + dimension: + is_time: true ai_context: synonyms: - - "total revenue" - - "gross sales" - - "sales amount" + - "year" - - name: total_profit + # Declares temporal role via is_time without a datatype annotation. + # Both datatype and is_time are independently optional. + - name: d_quarter_name expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_net_profit) - description: Total net profit from store sales - datatype: Decimal + expression: d_quarter_name + description: Quarter name (e.g., 2024Q1) + dimension: + is_time: true ai_context: synonyms: - - "net profit" - - "total earnings" - - "profit" + - "quarter" + - "fiscal quarter" - - name: customer_lifetime_value + # Declares temporal role via is_time without a datatype annotation. + # Both datatype and is_time are independently optional. + - name: d_moy expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) - description: Average lifetime sales value per customer - datatype: Decimal + expression: d_moy + description: Month of year (1-12) + dimension: + is_time: true ai_context: synonyms: - - "CLV" - - "LTV" - - "customer value" - - "lifetime revenue" + - "month" + + # Dimension table: Customer + - name: customer + source: tpcds.public.customer + primary_key: [c_customer_sk] # Simple primary key + unique_keys: + - [c_customer_sk] # Simple key: single column + description: Customer dimension with demographic information + ai_context: + synonyms: + - "customers" + - "shoppers" + - "buyers" - - name: sales_by_brand + fields: + - name: c_customer_sk expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - description: Total sales by brand (requires grouping by item.i_brand) - datatype: Decimal + expression: c_customer_sk + description: Surrogate key for customer + datatype: Integer + dimension: + is_time: false + + - name: c_customer_id + expression: + dialects: + - dialect: ANSI_SQL + expression: c_customer_id + description: Business key for customer + datatype: String + dimension: + is_time: false ai_context: synonyms: - - "brand sales" - - "brand performance" - - "brand revenue" + - "customer ID" + - "customer number" - - name: store_productivity + - name: c_first_name expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) - description: Sales per employee across stores - datatype: Decimal + expression: c_first_name + description: Customer first name + datatype: String + dimension: + is_time: false + + - name: c_last_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_last_name + description: Customer last name + datatype: String + dimension: + is_time: false + + - name: customer_full_name + expression: + dialects: + - dialect: ANSI_SQL + expression: c_first_name || ' ' || c_last_name + description: Customer full name (computed field) + datatype: String + dimension: + is_time: false ai_context: synonyms: - - "sales per employee" - - "employee productivity" - - "revenue per employee" + - "full name" + - "customer name" - # Window-function metrics. Each one wraps an aggregate in a window, so the - # grain named in the description is the GROUP BY the query is expected to supply. - - name: cumulative_sales + - name: c_email_address expression: dialects: - dialect: ANSI_SQL - expression: SUM(SUM(store_sales.ss_ext_sales_price)) OVER (ORDER BY date_dim.d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) - description: Running total of sales revenue by date (requires grouping by date_dim.d_date) - datatype: Decimal + expression: c_email_address + description: Customer email address + datatype: String + dimension: + is_time: false ai_context: synonyms: - - "running total sales" - - "cumulative revenue" - - "sales to date" + - "email" + - "contact" + + # Dimension table: Item (Product) + - name: item + source: tpcds.public.item + primary_key: [i_item_sk] # Simple primary key + unique_keys: + - [i_item_sk] # Simple key: single column + description: Item/Product dimension with product attributes + ai_context: + synonyms: + - "products" + - "items" + - "merchandise" - - name: brand_rank_in_store + fields: + - name: i_item_sk expression: dialects: - dialect: ANSI_SQL - expression: RANK() OVER (PARTITION BY store.s_store_sk ORDER BY SUM(store_sales.ss_ext_sales_price) DESC) - description: Rank of each brand by sales within a store, 1 = highest (requires grouping by store.s_store_sk and item.i_brand) + expression: i_item_sk + description: Surrogate key for item datatype: Integer + dimension: + is_time: false + + - name: i_item_id + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_id + description: Business key for item + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - "item ID" + - "product ID" + - "SKU" + + - name: i_item_desc + expression: + dialects: + - dialect: ANSI_SQL + expression: i_item_desc + description: Item description + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - "product description" + - "item name" + + - name: i_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: i_brand + description: Brand name + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - "brand" + - "manufacturer" + + - name: i_category + expression: + dialects: + - dialect: ANSI_SQL + expression: i_category + description: Item category + datatype: String + dimension: + is_time: false ai_context: synonyms: - - "brand ranking by store" - - "top brands per store" - - "store brand leaderboard" + - "product category" + - "department" - - name: monthly_sales_change + - name: i_current_price expression: dialects: - dialect: ANSI_SQL - expression: SUM(store_sales.ss_ext_sales_price) - LAG(SUM(store_sales.ss_ext_sales_price), 1) OVER (ORDER BY date_dim.d_year, date_dim.d_moy) - description: Sales revenue change versus the previous month (requires grouping by date_dim.d_year and date_dim.d_moy) + expression: i_current_price + description: Current price of the item datatype: Decimal + dimension: + is_time: false + ai_context: + synonyms: + - "price" + - "list price" + + # Dimension table: Store + - name: store + source: tpcds.public.store + primary_key: [s_store_sk] # Simple primary key + unique_keys: + - [s_store_id] # Simple key: single column + description: Store dimension with location and store attributes + ai_context: + synonyms: + - "stores" + - "retail locations" + - "branches" + + fields: + - name: s_store_sk + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_sk + description: Surrogate key for store + datatype: Integer + dimension: + is_time: false + + - name: s_store_id + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_id + description: Business key for store + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - "store ID" + - "store number" + + - name: s_store_name + expression: + dialects: + - dialect: ANSI_SQL + expression: s_store_name + description: Store name + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - "store name" + - "location name" + + - name: s_city + expression: + dialects: + - dialect: ANSI_SQL + expression: s_city + description: City where store is located + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - "city" + - "location" + + - name: s_state + expression: + dialects: + - dialect: ANSI_SQL + expression: s_state + description: State where store is located + datatype: String + dimension: + is_time: false + ai_context: + synonyms: + - "state" + - "region" + + - name: s_number_employees + expression: + dialects: + - dialect: ANSI_SQL + expression: s_number_employees + description: Number of employees at the store + datatype: Integer ai_context: synonyms: - - "month over month sales change" - - "MoM sales" - - "sales delta" - - custom_extensions: - - vendor_name: SALESFORCE - data: | - { - "tableau_workbook_id": "tpcds_retail_dashboard", - "einstein_enabled": true, - "crm_sync": { - "enabled": true, - "sync_frequency": "daily", - "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" - }, - "tableau_semantics": { - "published": true, - "version": "0.1.1" - } - } - - - vendor_name: DBT - data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' + - "employee count" + - "staff size" + +# Relationships between datasets +relationships: + - name: store_sales_to_date + from: store_sales + to: date_dim + from_columns: [ss_sold_date_sk] + to_columns: [d_date_sk] + ai_context: + synonyms: + - "sales date relationship" + - "when sale occurred" + + - name: store_sales_to_customer + from: store_sales + to: customer + from_columns: [ss_customer_sk] + to_columns: [c_customer_sk] + ai_context: + synonyms: + - "customer purchase relationship" + - "who bought" + + - name: store_sales_to_item + from: store_sales + to: item + from_columns: [ss_item_sk] + to_columns: [i_item_sk] + ai_context: + synonyms: + - "product sold relationship" + - "what was sold" + + - name: store_sales_to_store + from: store_sales + to: store + from_columns: [ss_store_sk] + to_columns: [s_store_sk] + ai_context: + synonyms: + - "store location relationship" + - "where sale occurred" + +# Semantic model-level metrics spanning multiple datasets +metrics: + - name: total_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales revenue across all transactions + datatype: Decimal + ai_context: + synonyms: + - "total revenue" + - "gross sales" + - "sales amount" + + - name: total_profit + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_net_profit) + description: Total net profit from store sales + datatype: Decimal + ai_context: + synonyms: + - "net profit" + - "total earnings" + - "profit" + + - name: customer_lifetime_value + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / COUNT(DISTINCT customer.c_customer_sk) + description: Average lifetime sales value per customer + datatype: Decimal + ai_context: + synonyms: + - "CLV" + - "LTV" + - "customer value" + - "lifetime revenue" + + - name: sales_by_brand + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) + description: Total sales by brand (requires grouping by item.i_brand) + datatype: Decimal + ai_context: + synonyms: + - "brand sales" + - "brand performance" + - "brand revenue" + + - name: store_productivity + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) / NULLIF(SUM(store.s_number_employees), 0) + description: Sales per employee across stores + datatype: Decimal + ai_context: + synonyms: + - "sales per employee" + - "employee productivity" + - "revenue per employee" + + # Window-function metrics. Each one wraps an aggregate in a window, so the + # grain named in the description is the GROUP BY the query is expected to supply. + - name: cumulative_sales + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(SUM(store_sales.ss_ext_sales_price)) OVER (ORDER BY date_dim.d_date ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + description: Running total of sales revenue by date (requires grouping by date_dim.d_date) + datatype: Decimal + ai_context: + synonyms: + - "running total sales" + - "cumulative revenue" + - "sales to date" + + - name: brand_rank_in_store + expression: + dialects: + - dialect: ANSI_SQL + expression: RANK() OVER (PARTITION BY store.s_store_sk ORDER BY SUM(store_sales.ss_ext_sales_price) DESC) + description: Rank of each brand by sales within a store, 1 = highest (requires grouping by store.s_store_sk and item.i_brand) + datatype: Integer + ai_context: + synonyms: + - "brand ranking by store" + - "top brands per store" + - "store brand leaderboard" + + - name: monthly_sales_change + expression: + dialects: + - dialect: ANSI_SQL + expression: SUM(store_sales.ss_ext_sales_price) - LAG(SUM(store_sales.ss_ext_sales_price), 1) OVER (ORDER BY date_dim.d_year, date_dim.d_moy) + description: Sales revenue change versus the previous month (requires grouping by date_dim.d_year and date_dim.d_moy) + datatype: Decimal + ai_context: + synonyms: + - "month over month sales change" + - "MoM sales" + - "sales delta" + +custom_extensions: + - vendor_name: SALESFORCE + data: | + { + "tableau_workbook_id": "tpcds_retail_dashboard", + "einstein_enabled": true, + "crm_sync": { + "enabled": true, + "sync_frequency": "daily", + "customer_mapping": "customer.c_customer_id -> Account.AccountNumber" + }, + "tableau_semantics": { + "published": true, + "version": "0.1.1" + } + } + + - vendor_name: DBT + data: '{"project_name": "tpcds_analytics", "models_path": "models/semantic"}' diff --git a/validation/test_validate.py b/validation/test_validate.py index 156022a4..2519b955 100644 --- a/validation/test_validate.py +++ b/validation/test_validate.py @@ -305,15 +305,47 @@ def run_validator(self, content: str) -> subprocess.CompletedProcess[str]: text=True, ) + def test_invalid_document_roots_report_schema_errors(self): + for content in ("", "null\n", "[]\n", "42\n"): + with self.subTest(content=content): + result = self.run_validator(content) + + self.assertEqual(result.returncode, 1) + self.assertIn("Validation FAILED", result.stdout) + self.assertIn("[Schema]", result.stdout) + self.assertNotIn("Traceback", result.stderr) + + def test_wrapped_models_report_schema_errors(self): + model = {"name": "sales", "datasets": [{"name": "orders", "source": "orders"}]} + for wrapped in (model, [], [model], [model, model], None): + with self.subTest(semantic_model=wrapped): + content = yaml.safe_dump({"version": "0.2.0.dev0", "semantic_model": wrapped}) + result = self.run_validator(content) + + self.assertEqual(result.returncode, 1) + self.assertIn("Validation FAILED", result.stdout) + self.assertIn("[Schema]", result.stdout) + self.assertNotIn("Traceback", result.stderr) + + def test_malformed_datasets_report_schema_errors(self): + for datasets in (None, {}, "orders", [None]): + with self.subTest(datasets=datasets): + content = yaml.safe_dump({"version": "0.2.0.dev0", "name": "sales", "datasets": datasets}) + result = self.run_validator(content) + + self.assertEqual(result.returncode, 1) + self.assertIn("Validation FAILED", result.stdout) + self.assertIn("[Schema]", result.stdout) + self.assertNotIn("Traceback", result.stderr) + def test_duplicate_key_exits_nonzero(self): result = self.run_validator( "version: 0.2.0.dev0\n" - "semantic_model:\n" - " - name: sales\n" - " name: finance\n" - " datasets:\n" - " - name: orders\n" - " source: analytics.orders\n" + "name: sales\n" + "name: finance\n" + "datasets:\n" + " - name: orders\n" + " source: analytics.orders\n" ) self.assertEqual(result.returncode, 1) @@ -323,28 +355,66 @@ def test_duplicate_key_exits_nonzero(self): def test_valid_model_still_passes(self): result = self.run_validator( "version: 0.2.0.dev0\n" - "semantic_model:\n" - " - name: sales\n" - " datasets:\n" - " - name: orders\n" - " source: analytics.orders\n" + "name: sales\n" + "datasets:\n" + " - name: orders\n" + " source: analytics.orders\n" ) self.assertEqual(result.returncode, 0) self.assertIn("Validation PASSED", result.stdout) + def test_relationship_column_counts_are_checked_in_flat_documents(self): + cases = ( + (["customer_id"], ["id"], 0), + (["customer_id", "region_id"], ["id"], 1), + (["customer_id"], ["id", "region_id"], 1), + ) + for from_columns, to_columns, expected_code in cases: + with self.subTest(from_columns=from_columns, to_columns=to_columns): + document = { + "version": "0.2.0.dev0", + "name": "sales", + "datasets": [ + {"name": "orders", "source": "analytics.orders"}, + { + "name": "customers", + "source": "analytics.customers", + "primary_key": ["id"], + }, + ], + "relationships": [ + { + "name": "orders_to_customers", + "from": "orders", + "to": "customers", + "from_columns": from_columns, + "to_columns": to_columns, + } + ], + } + result = self.run_validator(yaml.safe_dump(document)) + + self.assertEqual(result.returncode, expected_code) + self.assertNotIn("Traceback", result.stderr) + if expected_code: + self.assertIn("Validation FAILED", result.stdout) + self.assertIn("[Arity]", result.stdout) + self.assertIn("must have the same number of columns", result.stdout) + else: + self.assertIn("Validation PASSED", result.stdout) + def test_root_dialects_and_vendors_are_rejected(self): - # The document root is version and semantic_model only; the dialect and - # vendor enumerations belong under expression.dialects and custom_extensions. + # Dialects and vendors belong under expression.dialects and custom_extensions, + # not alongside the model properties at the document root. result = self.run_validator( "version: 0.2.0.dev0\n" "dialects: [ANSI_SQL]\n" "vendors: [DBT]\n" - "semantic_model:\n" - " - name: sales\n" - " datasets:\n" - " - name: orders\n" - " source: analytics.orders\n" + "name: sales\n" + "datasets:\n" + " - name: orders\n" + " source: analytics.orders\n" ) self.assertEqual(result.returncode, 1) diff --git a/validation/tests/test_validate.py b/validation/tests/test_validate.py index bc41135b..1bc826f3 100644 --- a/validation/tests/test_validate.py +++ b/validation/tests/test_validate.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import json from importlib.util import module_from_spec, spec_from_file_location from pathlib import Path @@ -35,16 +36,18 @@ validate_relationship_column_arity = _VALIDATE.validate_relationship_column_arity +@pytest.fixture +def core_schema() -> dict: + schema_path = Path(__file__).parents[2] / "core-spec" / "ossie-schema.json" + return json.loads(schema_path.read_text()) + + def _document(datasets: list[dict], relationships: list[dict]) -> dict: return { "version": "0.2.0.dev0", - "semantic_model": [ - { - "name": "m", - "datasets": datasets, - "relationships": relationships, - } - ], + "name": "m", + "datasets": datasets, + "relationships": relationships, } @@ -58,6 +61,131 @@ def _document(datasets: list[dict], relationships: list[dict]) -> dict: _ORDERS = {"name": "orders", "source": "db.s.orders"} +def test_accepts_a_single_root_model(core_schema: dict) -> None: + document = _document([_ORDERS, _CUSTOMERS], []) + + assert _VALIDATE.validate_schema(document, core_schema) == [] + + +def test_rejects_empty_root_datasets(core_schema: dict) -> None: + errors = _VALIDATE.validate_schema(_document([], []), core_schema) + + assert errors == ["[Schema] datasets: [] should be non-empty"] + + +def test_embedded_semantic_model_does_not_require_document_version(core_schema: dict) -> None: + # Ontology components reference this definition without a document envelope. + embedded_schema = { + "$ref": "#/$defs/SemanticModel", + "$defs": core_schema["$defs"], + } + model = _document([_ORDERS, _CUSTOMERS], []) + del model["version"] + + assert _VALIDATE.validate_schema(model, embedded_schema) == [] + + +@pytest.mark.parametrize("unknown_property", ["dataset", "owner", "dialects", "vendors"]) +def test_rejects_unknown_root_properties(core_schema: dict, unknown_property: str) -> None: + document = _document([_ORDERS], []) + document[unknown_property] = "unexpected" + + errors = _VALIDATE.validate_schema(document, core_schema) + + assert any( + "Additional properties are not allowed" in error and unknown_property in error + for error in errors + ) + + +@pytest.mark.parametrize("required_property", ["version", "name", "datasets"]) +def test_requires_model_and_document_properties(core_schema: dict, required_property: str) -> None: + document = _document([_ORDERS], []) + del document[required_property] + + errors = _VALIDATE.validate_schema(document, core_schema) + + assert any(f"'{required_property}' is a required property" in error for error in errors) + + +@pytest.mark.parametrize("property_name", ["name", "datasets"]) +def test_rejects_null_model_properties(core_schema: dict, property_name: str) -> None: + document = _document([_ORDERS], []) + document[property_name] = None + + errors = _VALIDATE.validate_schema(document, core_schema) + + assert any(f"[Schema] {property_name}:" in error for error in errors) + + +@pytest.mark.parametrize( + "wrapped", + [ + None, + [], + {"name": "one", "datasets": []}, + [{"name": "one", "datasets": []}], + [{"name": "one", "datasets": []}, {"name": "two", "datasets": []}], + ], +) +def test_rejects_legacy_or_object_wrappers(core_schema: dict, wrapped: object) -> None: + document = {"version": "0.2.0.dev0", "semantic_model": wrapped} + + assert _VALIDATE.validate_schema(document, core_schema) + + # A wrapper must also be rejected when a valid root model is present. + document.update(_document([_ORDERS], [])) + errors = _VALIDATE.validate_schema(document, core_schema) + + assert any( + "Additional properties are not allowed" in error and "semantic_model" in error + for error in errors + ) + + +@pytest.mark.parametrize( + "data", + [ + None, + [], + 42, + {"version": "0.2.0.dev0"}, + {"version": "0.2.0.dev0", "semantic_model": [{"name": "old", "datasets": []}]}, + ], +) +def test_semantic_checks_skip_non_model_payloads(data: object) -> None: + assert _VALIDATE.validate_unique_names(data) == [] + assert validate_references(data) == [] + assert validate_relationship_column_arity(data) == [] + assert _VALIDATE.validate_sql(data) == [] + + +def test_unique_names_are_checked_in_the_root_model() -> None: + errors = _VALIDATE.validate_unique_names(_document([_ORDERS, _ORDERS], [])) + + assert errors == ["[Unique] Duplicate dataset name 'orders' in model 'm'"] + + +def test_sql_checks_traverse_root_fields_and_metrics(monkeypatch: pytest.MonkeyPatch) -> None: + seen = [] + + def record_expression(expression: str, dialect: str, context: str) -> None: + seen.append((expression, dialect, context)) + + monkeypatch.setattr(_VALIDATE, "SQLGLOT_AVAILABLE", True) + monkeypatch.setattr(_VALIDATE, "validate_sql_expression", record_expression) + expression = {"dialects": [{"dialect": "ANSI_SQL", "expression": "value"}]} + dataset = {**_ORDERS, "fields": [{"name": "value", "expression": expression}]} + document = _document([dataset], []) + document["metrics"] = [{"name": "total", "expression": expression}] + + assert _VALIDATE.validate_sql(document) == [] + assert seen == [ + ("value", "ANSI_SQL", "Field 'orders.value' in model 'm' (ANSI_SQL)"), + ("value", "ANSI_SQL", "Metric 'total' in model 'm' (ANSI_SQL)"), + ] + + def _relationship(to_columns: list[str], to: str = "customers") -> dict: return { "name": "orders_to_customers", diff --git a/validation/validate.py b/validation/validate.py index d4586528..73eb2ca0 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -29,7 +29,7 @@ """ Ossie Semantic Model Validator -Validates Ossie YAML files against: +Validates Ossie YAML or JSON documents containing one semantic model against: 1. JSON Schema (structure, types, enums) 2. Unique names (datasets, fields, metrics, relationships) 3. Valid relationship references @@ -164,32 +164,35 @@ def find_duplicates(items: list[str]) -> list[str]: def validate_unique_names(data: dict) -> list[str]: """Validate unique names for datasets, fields, metrics, relationships.""" + if not isinstance(data, dict) or "datasets" not in data: + return [] + + model = data errors = [] - for model in data.get("semantic_model", []): - model_name = model.get("name", "") + model_name = model.get("name", "") - # Check unique dataset names - dataset_names = [d.get("name") for d in model.get("datasets", []) if d.get("name")] - for dup in find_duplicates(dataset_names): - errors.append(f"[Unique] Duplicate dataset name '{dup}' in model '{model_name}'") + # Check unique dataset names + dataset_names = [d.get("name") for d in model.get("datasets", []) if d.get("name")] + for dup in find_duplicates(dataset_names): + errors.append(f"[Unique] Duplicate dataset name '{dup}' in model '{model_name}'") - # Check unique field names within each dataset - for dataset in model.get("datasets", []): - dataset_name = dataset.get("name", "") - field_names = [f.get("name") for f in dataset.get("fields", []) if f.get("name")] - for dup in find_duplicates(field_names): - errors.append(f"[Unique] Duplicate field name '{dup}' in dataset '{dataset_name}'") + # Check unique field names within each dataset + for dataset in model.get("datasets", []): + dataset_name = dataset.get("name", "") + field_names = [f.get("name") for f in dataset.get("fields", []) if f.get("name")] + for dup in find_duplicates(field_names): + errors.append(f"[Unique] Duplicate field name '{dup}' in dataset '{dataset_name}'") - # Check unique metric names - metric_names = [m.get("name") for m in model.get("metrics", []) if m.get("name")] - for dup in find_duplicates(metric_names): - errors.append(f"[Unique] Duplicate metric name '{dup}' in model '{model_name}'") + # Check unique metric names + metric_names = [m.get("name") for m in model.get("metrics", []) if m.get("name")] + for dup in find_duplicates(metric_names): + errors.append(f"[Unique] Duplicate metric name '{dup}' in model '{model_name}'") - # Check unique relationship names - rel_names = [r.get("name") for r in model.get("relationships", []) if r.get("name")] - for dup in find_duplicates(rel_names): - errors.append(f"[Unique] Duplicate relationship name '{dup}' in model '{model_name}'") + # Check unique relationship names + rel_names = [r.get("name") for r in model.get("relationships", []) if r.get("name")] + for dup in find_duplicates(rel_names): + errors.append(f"[Unique] Duplicate relationship name '{dup}' in model '{model_name}'") return errors @@ -197,38 +200,41 @@ def validate_unique_names(data: dict) -> list[str]: def validate_references(data: dict) -> list[str]: """Validate that relationships reference existing datasets and that to_columns covers a declared key of the 'to' dataset.""" + if not isinstance(data, dict) or "datasets" not in data: + return [] + + model = data errors = [] - for model in data.get("semantic_model", []): - model_name = model.get("name", "") - datasets = {d.get("name"): d for d in model.get("datasets", []) if d.get("name")} - - for rel in model.get("relationships", []): - rel_name = rel.get("name", "") - from_ds = rel.get("from") - to_ds = rel.get("to") - - if from_ds and from_ds not in datasets: - errors.append(f"[Reference] Relationship '{rel_name}' in model '{model_name}' references unknown dataset '{from_ds}'") - if to_ds and to_ds not in datasets: - errors.append(f"[Reference] Relationship '{rel_name}' in model '{model_name}' references unknown dataset '{to_ds}'") - - # The spec defines to_columns as "Primary/unique key columns in the - # 'to' dataset". Coverage (superset of a key) still guarantees the - # many-to-one join, and declared keys may be incomplete since - # primary_key and unique_keys are optional — so accept any - # to_columns that covers a declared key, report a warning rather - # than an error, and skip datasets that declare no keys. - # Shape guards keep semantic checks from crashing on documents - # that already fail schema validation. - dataset = datasets.get(to_ds) - to_columns = rel.get("to_columns") - if dataset and isinstance(to_columns, list) and to_columns: - candidate_keys = [dataset.get("primary_key")] + list(dataset.get("unique_keys") or []) - declared_keys = [k for k in candidate_keys if isinstance(k, list) and k] - to_column_set = set(to_columns) - if declared_keys and not any(set(key) <= to_column_set for key in declared_keys): - errors.append(f"[Reference] Warning: Relationship '{rel_name}' in model '{model_name}': to_columns {to_columns} does not cover the primary key or a unique key of dataset '{to_ds}'") + model_name = model.get("name", "") + datasets = {d.get("name"): d for d in model.get("datasets", []) if d.get("name")} + + for rel in model.get("relationships", []): + rel_name = rel.get("name", "") + from_ds = rel.get("from") + to_ds = rel.get("to") + + if from_ds and from_ds not in datasets: + errors.append(f"[Reference] Relationship '{rel_name}' in model '{model_name}' references unknown dataset '{from_ds}'") + if to_ds and to_ds not in datasets: + errors.append(f"[Reference] Relationship '{rel_name}' in model '{model_name}' references unknown dataset '{to_ds}'") + + # The spec defines to_columns as "Primary/unique key columns in the + # 'to' dataset". Coverage (superset of a key) still guarantees the + # many-to-one join, and declared keys may be incomplete since + # primary_key and unique_keys are optional — so accept any + # to_columns that covers a declared key, report a warning rather + # than an error, and skip datasets that declare no keys. + # Shape guards keep semantic checks from crashing on documents + # that already fail schema validation. + dataset = datasets.get(to_ds) + to_columns = rel.get("to_columns") + if dataset and isinstance(to_columns, list) and to_columns: + candidate_keys = [dataset.get("primary_key")] + list(dataset.get("unique_keys") or []) + declared_keys = [k for k in candidate_keys if isinstance(k, list) and k] + to_column_set = set(to_columns) + if declared_keys and not any(set(key) <= to_column_set for key in declared_keys): + errors.append(f"[Reference] Warning: Relationship '{rel_name}' in model '{model_name}': to_columns {to_columns} does not cover the primary key or a unique key of dataset '{to_ds}'") return errors @@ -239,26 +245,29 @@ def validate_relationship_column_arity(data: dict) -> list[str]: The spec requires the two arrays to correspond positionally, so their lengths must match. JSON Schema cannot express this, so it is checked here. """ + if not isinstance(data, dict) or "datasets" not in data: + return [] + + model = data errors = [] - for model in data.get("semantic_model", []): - model_name = model.get("name", "") + model_name = model.get("name", "") - for rel in model.get("relationships", []): - rel_name = rel.get("name", "") - from_columns = rel.get("from_columns") - to_columns = rel.get("to_columns") + for rel in model.get("relationships", []): + rel_name = rel.get("name", "") + from_columns = rel.get("from_columns") + to_columns = rel.get("to_columns") - # Skip anything that already failed schema validation. - if not isinstance(from_columns, list) or not isinstance(to_columns, list): - continue + # Skip anything that already failed schema validation. + if not isinstance(from_columns, list) or not isinstance(to_columns, list): + continue - if len(from_columns) != len(to_columns): - errors.append( - f"[Arity] Relationship '{rel_name}' in model '{model_name}': " - f"from_columns ({len(from_columns)}) and " - f"to_columns ({len(to_columns)}) must have the same number of columns" - ) + if len(from_columns) != len(to_columns): + errors.append( + f"[Arity] Relationship '{rel_name}' in model '{model_name}': " + f"from_columns ({len(from_columns)}) and " + f"to_columns ({len(to_columns)}) must have the same number of columns" + ) return errors @@ -290,46 +299,46 @@ def validate_sql_expression(expr: str, dialect: str, context: str) -> str | None def validate_sql(data: dict) -> list[str]: """Validate SQL expressions in fields and metrics.""" - # Only semantic model files contain SQL expressions to validate. - if not data.get("semantic_model"): + # Only core semantic model documents contain a root datasets property. + if not isinstance(data, dict) or "datasets" not in data: return [] if not SQLGLOT_AVAILABLE: return ["[SQL] Warning: sqlglot not installed, skipping SQL validation. Install with: pip install sqlglot"] + model = data errors = [] - for model in data.get("semantic_model", []): - model_name = model.get("name", "") - - # Validate field expressions - for dataset in model.get("datasets", []): - dataset_name = dataset.get("name", "") - for field in dataset.get("fields", []): - field_name = field.get("name", "") - expression = field.get("expression", {}) - for dialect_expr in expression.get("dialects", []): - dialect = dialect_expr.get("dialect", "ANSI_SQL") - expr = dialect_expr.get("expression", "") - if expr: - context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' ({dialect})" - error = validate_sql_expression(expr, dialect, context) - if error: - errors.append(error) - - # Validate metric expressions - for metric in model.get("metrics", []): - metric_name = metric.get("name", "") - expression = metric.get("expression", {}) + model_name = model.get("name", "") + + # Validate field expressions + for dataset in model.get("datasets", []): + dataset_name = dataset.get("name", "") + for field in dataset.get("fields", []): + field_name = field.get("name", "") + expression = field.get("expression", {}) for dialect_expr in expression.get("dialects", []): dialect = dialect_expr.get("dialect", "ANSI_SQL") expr = dialect_expr.get("expression", "") if expr: - context = f"Metric '{metric_name}' in model '{model_name}' ({dialect})" + context = f"Field '{dataset_name}.{field_name}' in model '{model_name}' ({dialect})" error = validate_sql_expression(expr, dialect, context) if error: errors.append(error) + # Validate metric expressions + for metric in model.get("metrics", []): + metric_name = metric.get("name", "") + expression = metric.get("expression", {}) + for dialect_expr in expression.get("dialects", []): + dialect = dialect_expr.get("dialect", "ANSI_SQL") + expr = dialect_expr.get("expression", "") + if expr: + context = f"Metric '{metric_name}' in model '{model_name}' ({dialect})" + error = validate_sql_expression(expr, dialect, context) + if error: + errors.append(error) + return errors @@ -372,8 +381,9 @@ def main(): errors = [] errors.extend(validate_schema(data, schema)) - # Run semantic-model-specific checks only for semantic model payloads. - if data.get("semantic_model"): + # Semantic checks rely on valid structure; let schema validation report + # malformed inputs (including legacy arrays) without traversing them. + if not errors and isinstance(data, dict) and "datasets" in data: errors.extend(validate_unique_names(data)) errors.extend(validate_references(data)) errors.extend(validate_relationship_column_arity(data))