diff --git a/converters/orionbelt/ossie_obml_mapping_analysis.md b/converters/orionbelt/ossie_obml_mapping_analysis.md index 957e9cf0..17a97f06 100644 --- a/converters/orionbelt/ossie_obml_mapping_analysis.md +++ b/converters/orionbelt/ossie_obml_mapping_analysis.md @@ -143,7 +143,8 @@ These OBML features have no direct Ossie equivalent. Where possible, metadata is - Measure `withinGroup` — preserved in metric `custom_extensions` (`obml_within_group`) - Metric `format` — preserved in metric `custom_extensions` (`obml_format`) - Locale settings — not yet preserved -- `abstractType` (OBML type system) — preserved in field `custom_extensions` (`obml_abstract_type`) +- `abstractType` (OBML type system): emitted as the spec field `datatype` (`json` → `Opaque`, `time_tz` → `Time`) and preserved exactly in field `custom_extensions` (`obml_abstract_type`) +- Measure/metric `dataType`: emitted as metric `datatype` when declared (`decimal(p, s)` → `Decimal`), exact value preserved via `obml_data_type` ### 2.6 Ossie-Specific Features and How They Map to OBML @@ -158,7 +159,7 @@ These OBML features have no direct Ossie equivalent. Where possible, metadata is ### 3.1 Ossie → OBML 1. Parse `source` string to extract `database`, `schema`, and `table` -2. Convert fields to columns with type inference (heuristic-based `abstractType`) +2. Convert fields to columns; `abstractType` comes from the spec `datatype` (`Decimal` narrows to `float`), then legacy `data_type`, then a name heuristic (also used for `Opaque`); metric `datatype` sets the exact measure/metric `dataType` (`Decimal` → the model's `settings.defaultNumericDataType`, else `decimal(18, 2)`). A stashed `obml_abstract_type` or `obml_data_type` is restored while it agrees with `datatype`; an edited `datatype` wins over it 3. Restructure global relationships into inline joins on data objects 4. Decompose metric SQL expressions into OBML measures + metrics 5. Extract dimension-flagged fields into the top-level `dimensions` section (excluding FK/PK join keys) diff --git a/converters/orionbelt/src/ossie_orionbelt/_common.py b/converters/orionbelt/src/ossie_orionbelt/_common.py index 9d4a5412..1987c384 100644 --- a/converters/orionbelt/src/ossie_orionbelt/_common.py +++ b/converters/orionbelt/src/ossie_orionbelt/_common.py @@ -91,3 +91,124 @@ "timestamp": "timestamp", "boolean": "boolean", } + +# ─── Ossie DataType ⇄ OBML ────────────────────────────────────────────────── +# Ossie `datatype` on Field/Metric is a *logical* type backed by the capitalised +# `DataType` enum in core-spec/ossie-schema.json - the same layer as OBML's column +# `abstractType` - so this is the field/dimension mapping. +# +# `Decimal` has no logical-layer equivalent in OBML: OBML models exact decimal at +# the physical/result layer (`sqlType`/`sqlPrecision`/`sqlScale`, measure/metric +# `dataType` via `decimal(p, s)`), not as a coarse `abstractType`. So `Decimal` +# narrows to `float` for fields, but is recovered exactly for metrics via the +# physical `dataType` map below (`OSSIE_DATATYPE_TO_OBML_PHYSICAL`). +# +# `Opaque` is Ossie's "known type outside the portable vocabulary" marker and is +# intentionally absent so it falls back to the name heuristic on import. +OSSIE_DATATYPE_TO_OBML_ABSTRACT = { + "String": "string", + "Integer": "int", + "Float": "float", + "Decimal": "float", + "Boolean": "boolean", + "Date": "date", + "Time": "time", + "DateTime": "timestamp", + "DateTimeTz": "timestamp_tz", +} + +# OBML column `abstractType` -> Ossie `DataType`, for the export direction. +OBML_ABSTRACT_TO_OSSIE_DATATYPE = { + "string": "String", + "json": "Opaque", + "int": "Integer", + "float": "Float", + "date": "Date", + "time": "Time", + "time_tz": "Time", + "timestamp": "DateTime", + "timestamp_tz": "DateTimeTz", + "boolean": "Boolean", +} + +# Metric/measure `datatype`. Unlike fields, OBML measures/metrics carry an exact +# `dataType` (physical vocabulary: `integer`/`double`/`decimal(p, s)`/...), which +# is where `Decimal` genuinely belongs. So Ossie metric `datatype` maps to that +# field, not the coarse `abstractType`. +OBML_DECIMAL_DEFAULT = "decimal(18, 2)" # mirrors OrionBelt's built-in default + +# Ossie `DataType` -> OBML physical `dataType` (import direction). `Opaque` is +# omitted (non-portable). `DateTimeTz` has no tz-aware physical form, so it +# narrows to `timestamp`. +OSSIE_DATATYPE_TO_OBML_PHYSICAL = { + "String": "string", + "Integer": "integer", + "Float": "double", + "Decimal": OBML_DECIMAL_DEFAULT, + "Boolean": "boolean", + "Date": "date", + "Time": "time", + "DateTime": "timestamp", + "DateTimeTz": "timestamp", +} + +# OBML physical `dataType` -> Ossie `DataType` (export direction). `decimal(p, s)` +# is handled by ``obml_datatype_to_ossie`` since it is parametrised. +OBML_PHYSICAL_TO_OSSIE_DATATYPE = { + "string": "String", + "integer": "Integer", + "bigint": "Integer", + "double": "Float", + "boolean": "Boolean", + "date": "Date", + "time": "Time", + "timestamp": "DateTime", +} + + +def obml_datatype_to_ossie(data_type: object) -> str | None: + """Map an explicit OBML measure/metric ``dataType`` to an Ossie ``DataType``. + + Returns ``None`` when there is no mapping, so the caller emits nothing rather + than an unknown type. ``decimal(p, s)`` maps to ``Decimal``. A hand-authored + document may carry a non-string ``dataType`` (``123``) that no schema check + has rejected yet; that has no mapping either, rather than aborting the whole + conversion. + """ + if not isinstance(data_type, str): + return None + normalized = data_type.strip().lower() + if not normalized: + return None + if normalized.startswith("decimal"): + return "Decimal" + return OBML_PHYSICAL_TO_OSSIE_DATATYPE.get(normalized) + + +def obml_decimal_default(settings: object) -> str: + """The ``dataType`` an Ossie ``Decimal`` metric becomes in this model. + + OBML lets a model set ``settings.defaultNumericDataType`` (always a + ``decimal(p, s)``, which OrionBelt enforces), and a model configured for + ``decimal(20, 6)`` should not have its metrics written as the built-in + ``decimal(18, 2)``. Anything other than a decimal string there falls back to + the built-in default. + """ + if isinstance(settings, dict): + configured = settings.get("defaultNumericDataType") + if isinstance(configured, str) and configured.strip().lower().startswith("decimal"): + return configured + return OBML_DECIMAL_DEFAULT + + +def ossie_metric_datatype_to_obml(ossie_datatype: object, decimal_default: str) -> str | None: + """Map an Ossie metric ``datatype`` to the OBML measure/metric ``dataType``. + + ``Decimal`` takes the model's numeric default; ``Opaque``, an unknown value + or a non-string has no mapping. + """ + if not isinstance(ossie_datatype, str): + return None + if ossie_datatype == "Decimal": + return decimal_default + return OSSIE_DATATYPE_TO_OBML_PHYSICAL.get(ossie_datatype) diff --git a/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py b/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py index d46687eb..1990f719 100644 --- a/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py +++ b/converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py @@ -32,7 +32,9 @@ _OSSIE_VENDOR_READ, _OSSIE_VERSION, _VENDOR_OBML, + OBML_ABSTRACT_TO_OSSIE_DATATYPE, OBML_TO_OSSIE_TYPE, + obml_datatype_to_ossie, ) @@ -385,8 +387,12 @@ def _convert_column( if ai_ctx: field["ai_context"] = ai_ctx - # Preserve OBML type info in custom_extensions for roundtrip fidelity + # Emit the spec `datatype` from the OBML abstractType so exported fields + # carry a portable logical type... abstract_type = col_obj.get("abstractType", "string") + field["datatype"] = OBML_ABSTRACT_TO_OSSIE_DATATYPE.get(abstract_type, "String") + # ...and stash the exact abstractType in custom_extensions so the return + # trip restores it verbatim, lossless through the narrowing map. ossie_type = OBML_TO_OSSIE_TYPE.get(abstract_type, "string") ext_data: dict[str, Any] = { "data_type": ossie_type, @@ -549,6 +555,7 @@ def _convert_measures_and_metrics( ossie_metric = self._convert_measure(measure_name, measure_obj, data_objects) if ossie_metric: self._carry_foreign_to_ossie_metric(measure_obj, ossie_metric) + self._emit_ossie_metric_datatype(measure_obj, ossie_metric) ossie_metrics.append(ossie_metric) # Convert OBML metrics (which reference measures) to Ossie metrics @@ -571,6 +578,7 @@ def _convert_measures_and_metrics( ) if ossie_metric: self._carry_foreign_to_ossie_metric(metric_obj, ossie_metric) + self._emit_ossie_metric_datatype(metric_obj, ossie_metric) ossie_metrics.append(ossie_metric) return ossie_metrics @@ -584,6 +592,18 @@ def _carry_foreign_to_ossie_metric(self, obml_obj: dict, ossie_metric: dict) -> if not ossie_metric["custom_extensions"]: del ossie_metric["custom_extensions"] + def _emit_ossie_metric_datatype(self, obml_obj: dict, ossie_metric: dict) -> None: + """Emit the spec `datatype` from an explicit OBML measure/metric `dataType`. + + Only fires when the OBML object declares an exact `dataType`, so plain + measures (whose type is only the defaulted `resultType`) stay untouched + and round trips stay idempotent. The exact `dataType` also round-trips via + `obml_data_type` in `custom_extensions`; this adds the portable field. + """ + ossie_dt = obml_datatype_to_ossie(obml_obj.get("dataType")) + if ossie_dt: + ossie_metric["datatype"] = ossie_dt + def _convert_measure(self, name: str, measure: dict, data_objects: dict) -> dict | None: """Convert an OBML measure to an Ossie metric.""" diff --git a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py index 600d38cb..b2cc5410 100644 --- a/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py +++ b/converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py @@ -34,7 +34,13 @@ _OSSIE_VERSION, _SQL_PARSEABLE_DIALECTS, _VENDOR_OSSIE, + OBML_ABSTRACT_TO_OSSIE_DATATYPE, + OBML_DECIMAL_DEFAULT, + OSSIE_DATATYPE_TO_OBML_ABSTRACT, OSSIE_TO_OBML_TYPE, + obml_datatype_to_ossie, + obml_decimal_default, + ossie_metric_datatype_to_obml, ) # A dataset/column identifier in a resolved metric expression: either a bare SQL @@ -58,6 +64,10 @@ def __init__( # or an expression our parser cannot decompose). Preserved verbatim # rather than dropped — see ``_preserve_unconverted_metric``. self._unconverted_metrics: list[dict] = [] + # What an Ossie ``Decimal`` metric becomes: the model's own + # ``settings.defaultNumericDataType`` when it carries one, set per model + # in ``convert``. + self._decimal_default = OBML_DECIMAL_DEFAULT def _normalize_legacy_v01(self) -> None: """Promote Ossie v0.1.x payloads to the v0.2 shape, in place. @@ -168,6 +178,7 @@ def convert(self) -> dict: # ── Measures & Metrics ────────────────────────────────────── ossie_metrics = model.get("metrics", []) + self._decimal_default = obml_decimal_default(self._stashed_obml_settings(model)) measures, metrics = self._convert_metrics(ossie_metrics, ds_map) if measures: obml["measures"] = measures @@ -211,6 +222,22 @@ def convert(self) -> dict: return obml + @staticmethod + def _stashed_obml_settings(model: dict) -> object: + """The OBML ``settings`` an OBML-origin model stashed on export, if any. + + Read ahead of the metrics, which need the numeric default; the model + properties themselves are restored after them, as before. + """ + for ext in model.get("custom_extensions", []): + if ext.get("vendor_name") in _OBML_VENDOR_READ: + try: + data = json.loads(ext.get("data", "{}")) + except (json.JSONDecodeError, TypeError): + return None + return data.get("obml_settings") if isinstance(data, dict) else None + return None + @staticmethod def _carry_foreign_extensions(ossie_exts: list[dict] | None, obml_target: dict[str, Any]) -> None: """Carry third-party Ossie custom_extensions verbatim into OBML. @@ -374,10 +401,22 @@ def _convert_field(self, field: dict) -> tuple[str, dict]: elif code == name and dialects: code = dialects[0].get("expression", name) - # Determine abstract type: prefer explicit data_type, fall back to heuristic - ossie_type = field.get("data_type", "") - if ossie_type and ossie_type in OSSIE_TO_OBML_TYPE: - abstract_type = OSSIE_TO_OBML_TYPE[ossie_type] + # Determine abstract type. Precedence: the spec `datatype` (capitalised + # `DataType` enum) > legacy lowercase `data_type` > name heuristic. An + # OBML-origin field additionally restores its exact `abstractType` from + # the stashed extension below, keeping OBML -> Ossie -> OBML lossless, + # unless the `datatype` was edited since. A non-string value (a + # hand-authored document) counts as absent rather than crashing. + ossie_datatype = field.get("datatype") + if not isinstance(ossie_datatype, str): + ossie_datatype = "" + legacy_type = field.get("data_type") + if not isinstance(legacy_type, str): + legacy_type = "" + if ossie_datatype in OSSIE_DATATYPE_TO_OBML_ABSTRACT: + abstract_type = OSSIE_DATATYPE_TO_OBML_ABSTRACT[ossie_datatype] + elif legacy_type and legacy_type in OSSIE_TO_OBML_TYPE: + abstract_type = OSSIE_TO_OBML_TYPE[legacy_type] else: abstract_type = self._infer_obml_type(field) @@ -411,6 +450,21 @@ def _convert_field(self, field: dict) -> tuple[str, dict]: if ext.get("vendor_name") in _OBML_VENDOR_READ: try: ext_data = json.loads(ext.get("data", "{}")) + # Restore the exact OBML abstractType stashed on export, so a + # narrowing datatype map (e.g. time_tz -> Time) never + # degrades an OBML-origin round trip. The stash yields to a + # `datatype` that no longer agrees with it: that is an edit + # made in Ossie after the export, and it is the newer fact. + stashed = ext_data.get("obml_abstract_type") + if isinstance(stashed, str) and stashed: + stashed_ossie = OBML_ABSTRACT_TO_OSSIE_DATATYPE.get(stashed) + edited = ( + ossie_datatype in OSSIE_DATATYPE_TO_OBML_ABSTRACT + and stashed_ossie is not None + and stashed_ossie != ossie_datatype + ) + if not edited: + col["abstractType"] = stashed if ext_data.get("obml_sql_type"): col["sqlType"] = ext_data["obml_sql_type"] if ext_data.get("obml_sql_precision") is not None: @@ -829,6 +883,21 @@ def _convert_metrics(self, ossie_metrics: list, ds_map: dict) -> tuple[dict, dic target = metrics.get(m["name"]) or measures.get(m["name"]) if target is not None: self._carry_foreign_extensions(m.get("custom_extensions"), target) + # Ossie metric `datatype` -> OBML exact `dataType` (its natural + # home; `Decimal` -> the model's decimal(p, s)). Opaque, unknown + # and non-string values have no mapping and change nothing. + ossie_dt = m.get("datatype") + obml_dt = ossie_metric_datatype_to_obml(ossie_dt, self._decimal_default) + if obml_dt is None: + continue + # A dataType restored from the OBML-origin stash is more exact + # than the map (`decimal(20, 6)`, `bigint`) and is kept while it + # still agrees with `datatype`. When it names a different type, + # `datatype` was edited in Ossie after the export, and the edit + # wins over the stale stash. + stashed_ossie = obml_datatype_to_ossie(target.get("dataType")) + if stashed_ossie is None or stashed_ossie != ossie_dt: + target["dataType"] = obml_dt return measures, metrics diff --git a/converters/orionbelt/tests/test_ossie_converter_datatype.py b/converters/orionbelt/tests/test_ossie_converter_datatype.py new file mode 100644 index 00000000..752ec809 --- /dev/null +++ b/converters/orionbelt/tests/test_ossie_converter_datatype.py @@ -0,0 +1,335 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Ossie ``datatype`` support on the field conversion path. + +Ossie defines a first-class ``datatype`` on ``Field``/``Metric`` backed by a +capitalised ``DataType`` enum. The converter reads it (over the name heuristic) +on import and emits it on export. Because OBML's ``abstractType`` is a coarse +*logical* layer with no exact ``decimal``, ``Decimal`` narrows to ``float`` for +fields - so the exact ``abstractType`` is stashed for a lossless return trip. +""" + +from __future__ import annotations + +from typing import Any + +import ossie_orionbelt.converter as conv +from ossie_orionbelt._common import obml_datatype_to_ossie, obml_decimal_default + + +def _ossie_field(name: str, **extra: Any) -> dict[str, Any]: + field: dict[str, Any] = { + "name": name, + "expression": {"dialects": [{"dialect": "ANSI_SQL", "expression": name}]}, + } + field.update(extra) + return field + + +def _ossie_model(fields: list[dict[str, Any]]) -> dict[str, Any]: + return { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "sales", + "datasets": [ + {"name": "Orders", "source": "ANALYTICS.PUBLIC.ORDERS", "fields": fields} + ], + } + ], + } + + +def _obml_columns(obml: dict[str, Any]) -> dict[str, dict[str, Any]]: + (obj,) = obml["dataObjects"].values() + return obj["columns"] + + +def _ossie_fields(ossie: dict[str, Any]) -> dict[str, dict[str, Any]]: + out: dict[str, dict[str, Any]] = {} + for ds in ossie["semantic_model"][0]["datasets"]: + for f in ds.get("fields", []): + out[f["name"]] = f + return out + + +class TestImportDatatype: + """Ossie ``datatype`` maps to OBML ``abstractType`` on import.""" + + def test_direct_mappings(self) -> None: + ossie = _ossie_model( + [ + _ossie_field("s", datatype="String"), + _ossie_field("i", datatype="Integer"), + _ossie_field("f", datatype="Float"), + _ossie_field("b", datatype="Boolean"), + _ossie_field("d", datatype="Date"), + _ossie_field("t", datatype="Time"), + _ossie_field("dt", datatype="DateTime"), + _ossie_field("dttz", datatype="DateTimeTz"), + ] + ) + cols = _obml_columns(conv.OssietoOBML(ossie).convert()) + got = {name: c["abstractType"] for name, c in cols.items()} + assert got == { + "s": "string", + "i": "int", + "f": "float", + "b": "boolean", + "d": "date", + "t": "time", + "dt": "timestamp", + "dttz": "timestamp_tz", + } + + def test_decimal_narrows_to_float(self) -> None: + cols = _obml_columns( + conv.OssietoOBML(_ossie_model([_ossie_field("d", datatype="Decimal")])).convert() + ) + assert cols["d"]["abstractType"] == "float" + + def test_opaque_falls_back_to_heuristic(self) -> None: + # `price` is a heuristic float keyword; Opaque must not override it to a + # literal type - it means "unknown/non-portable", so the heuristic runs. + cols = _obml_columns( + conv.OssietoOBML(_ossie_model([_ossie_field("price", datatype="Opaque")])).convert() + ) + assert cols["price"]["abstractType"] == "float" + + def test_datatype_wins_over_legacy_and_heuristic(self) -> None: + # New capitalised `datatype` beats the legacy lowercase `data_type` and + # the name heuristic (name `amount` would heuristically be float). + ossie = _ossie_model([_ossie_field("amount", datatype="Integer", data_type="number")]) + cols = _obml_columns(conv.OssietoOBML(ossie).convert()) + assert cols["amount"]["abstractType"] == "int" + + +class TestExportDatatype: + """OBML ``abstractType`` emits a first-class Ossie ``datatype`` on export.""" + + @staticmethod + def _obml(abstract_type: str) -> dict[str, Any]: + return { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Val": {"code": "val", "abstractType": abstract_type}}, + } + } + } + + def test_emits_capitalised_datatype(self) -> None: + for abstract_type, expected in [ + ("int", "Integer"), + ("float", "Float"), + ("timestamp_tz", "DateTimeTz"), + ("json", "Opaque"), + ("boolean", "Boolean"), + ]: + ossie = conv.OBMLtoOssie(self._obml(abstract_type), model_name="s").convert() + assert _ossie_fields(ossie)["val"]["datatype"] == expected + + +class TestRoundtripLossless: + """OBML -> Ossie -> OBML preserves the exact abstractType via the stash.""" + + def test_narrowing_types_survive(self) -> None: + # json -> Opaque and time_tz -> Time are lossy in the datatype map alone; + # the stashed obml_abstract_type must restore them exactly. + for abstract_type in ["json", "time_tz", "float", "timestamp_tz"]: + obml = { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Val": {"code": "val", "abstractType": abstract_type}}, + } + } + } + back = conv.OssietoOBML(conv.OBMLtoOssie(obml, model_name="s").convert()).convert() + # The column key round-trips to its code (`val`); assert on the + # single column's abstractType regardless of its restored key. + (col,) = _obml_columns(back).values() + assert col["abstractType"] == abstract_type + + +def _ossie_model_with_metric(datatype: str) -> dict[str, Any]: + return { + "version": "0.2.0.dev0", + "semantic_model": [ + { + "name": "sales", + "datasets": [ + { + "name": "Orders", + "source": "A.P.ORDERS", + "fields": [_ossie_field("amount")], + } + ], + "metrics": [ + { + "name": "Total", + "expression": { + "dialects": [ + {"dialect": "ANSI_SQL", "expression": "SUM(Orders.amount)"} + ] + }, + "datatype": datatype, + } + ], + } + ], + } + + +class TestMetricDatatype: + """Ossie metric ``datatype`` maps to the exact OBML ``dataType`` and round-trips.""" + + def test_import_sets_exact_data_type(self) -> None: + for ossie_dt, expected in [ + ("Decimal", "decimal(18, 2)"), + ("Integer", "integer"), + ("Float", "double"), + ]: + obml = conv.OssietoOBML(_ossie_model_with_metric(ossie_dt)).convert() + # SUM(Orders.amount) becomes an OBML measure named "Total". + assert obml["measures"]["Total"]["dataType"] == expected + + def test_roundtrip_preserves_metric_datatype(self) -> None: + # Regression: Ossie -> OBML -> Ossie used to drop the metric datatype. + for ossie_dt in ["Decimal", "Integer", "Float"]: + ossie = _ossie_model_with_metric(ossie_dt) + back = conv.OBMLtoOssie(conv.OssietoOBML(ossie).convert(), model_name="sales").convert() + metric = back["semantic_model"][0]["metrics"][0] + assert metric.get("datatype") == ossie_dt + + def test_plain_measure_emits_no_datatype(self) -> None: + # A measure with no explicit dataType (only the defaulted resultType) + # must not gain a datatype on export - keeps round trips idempotent. + obml = { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Amount": {"code": "amount", "abstractType": "float"}}, + } + }, + "measures": { + "Total": { + "columns": [{"dataObject": "Orders", "column": "Amount"}], + "resultType": "float", + "aggregation": "sum", + } + }, + } + ossie = conv.OBMLtoOssie(obml, model_name="s").convert() + metric = ossie["semantic_model"][0]["metrics"][0] + assert "datatype" not in metric + + +def _obml_with_measure(**measure: Any) -> dict[str, Any]: + return { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Amount": {"code": "amount", "abstractType": "float"}}, + } + }, + "measures": { + "Total": { + "columns": [{"dataObject": "Orders", "column": "Amount"}], + "resultType": "float", + "aggregation": "sum", + **measure, + } + }, + } + + +def _only_metric(ossie: dict[str, Any]) -> dict[str, Any]: + (metric,) = ossie["semantic_model"][0]["metrics"] + return metric + + +class TestMalformedDatatype: + """A hand-authored document with a non-string type must not abort conversion.""" + + def test_non_string_obml_data_type_emits_nothing(self) -> None: + assert obml_datatype_to_ossie(123) is None + assert obml_datatype_to_ossie(" ") is None + ossie = conv.OBMLtoOssie(_obml_with_measure(dataType=123), model_name="s").convert() + assert "datatype" not in _only_metric(ossie) + + def test_non_string_ossie_datatype_is_ignored(self) -> None: + # `price` is a heuristic float keyword, so the field falls back to it. + ossie = _ossie_model_with_metric("Decimal") + model = ossie["semantic_model"][0] + model["datasets"][0]["fields"] = [_ossie_field("price", datatype=["Integer"])] + model["metrics"][0]["datatype"] = 7 + model["metrics"][0]["expression"]["dialects"][0]["expression"] = "SUM(Orders.price)" + obml = conv.OssietoOBML(ossie).convert() + assert _obml_columns(obml)["price"]["abstractType"] == "float" + assert "dataType" not in obml["measures"]["Total"] + + +class TestDecimalDefaultFromSettings: + """``Decimal`` follows the model's ``settings.defaultNumericDataType``.""" + + def test_model_default_is_used(self) -> None: + obml = _obml_with_measure() + obml["settings"] = {"defaultNumericDataType": "decimal(20, 6)"} + ossie = conv.OBMLtoOssie(obml, model_name="s").convert() + _only_metric(ossie)["datatype"] = "Decimal" + back = conv.OssietoOBML(ossie).convert() + assert back["measures"]["Total"]["dataType"] == "decimal(20, 6)" + assert back["settings"] == {"defaultNumericDataType": "decimal(20, 6)"} + + def test_builtin_default_without_or_with_an_invalid_setting(self) -> None: + assert obml_decimal_default(None) == "decimal(18, 2)" + assert obml_decimal_default({"defaultNumericDataType": "bigint"}) == "decimal(18, 2)" + assert obml_decimal_default({"defaultNumericDataType": 5}) == "decimal(18, 2)" + obml = conv.OssietoOBML(_ossie_model_with_metric("Decimal")).convert() + assert obml["measures"]["Total"]["dataType"] == "decimal(18, 2)" + + +class TestEditedDatatypeBeatsStaleStash: + """An Ossie ``datatype`` edited after an OBML export wins over the stash.""" + + def test_metric_edit_wins(self) -> None: + ossie = conv.OBMLtoOssie(_obml_with_measure(dataType="integer"), model_name="s").convert() + assert _only_metric(ossie)["datatype"] == "Integer" + _only_metric(ossie)["datatype"] = "Float" + back = conv.OssietoOBML(ossie).convert() + assert back["measures"]["Total"]["dataType"] == "double" + + def test_metric_stash_that_still_agrees_stays_exact(self) -> None: + for data_type in ["decimal(20, 6)", "bigint"]: + obml = _obml_with_measure(dataType=data_type) + back = conv.OssietoOBML(conv.OBMLtoOssie(obml, model_name="s").convert()).convert() + assert back["measures"]["Total"]["dataType"] == data_type + + def test_field_edit_wins(self) -> None: + obml = { + "dataObjects": { + "Orders": { + "code": "orders", + "columns": {"Val": {"code": "val", "abstractType": "timestamp_tz"}}, + } + } + } + ossie = conv.OBMLtoOssie(obml, model_name="s").convert() + _ossie_fields(ossie)["val"]["datatype"] = "DateTime" + (col,) = _obml_columns(conv.OssietoOBML(ossie).convert()).values() + assert col["abstractType"] == "timestamp" diff --git a/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py b/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py index d3fb61d0..5c97410f 100644 --- a/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py +++ b/converters/orionbelt/tests/test_ossie_metric_no_silent_loss.py @@ -380,3 +380,15 @@ def _unconverted_stash(obml: dict[str, Any]) -> list[dict[str, Any]]: if "obml_unconverted_metrics" in data: return data["obml_unconverted_metrics"] return [] + + +def test_explicit_datatype_roundtrips() -> None: + """A field's spec `datatype` survives both directions (apache/ossie#409).""" + _, col = conv.OssietoOBML(ossie={})._convert_field( + {"name": "total_amount", "datatype": "String"} + ) + assert col["abstractType"] == "string" + field = conv.OBMLtoOssie(obml={})._convert_column( + "total_amount", {"code": "total_amount", "abstractType": "string"}, "Orders", {} + ) + assert field.get("datatype") == "String"