fix(orionbelt): carry field and metric datatype in both directions

converters/orionbelt/ossie_obml_mapping_analysis.md

@@ -143,7 +143,8 @@ These OBML features have no direct Ossie equivalent. Where possible, metadata is
 - Measure `withinGroup` — preserved in metric `custom_extensions` (`obml_within_group`)
 - Metric `format` — preserved in metric `custom_extensions` (`obml_format`)
 - Locale settings — not yet preserved
-- `abstractType` (OBML type system) — preserved in field `custom_extensions` (`obml_abstract_type`)
+- `abstractType` (OBML type system): emitted as the spec field `datatype` (`json` → `Opaque`, `time_tz` → `Time`) and preserved exactly in field `custom_extensions` (`obml_abstract_type`)
+- Measure/metric `dataType`: emitted as metric `datatype` when declared (`decimal(p, s)` → `Decimal`), exact value preserved via `obml_data_type`
 
 ### 2.6 Ossie-Specific Features and How They Map to OBML
 
@@ -158,7 +159,7 @@ These OBML features have no direct Ossie equivalent. Where possible, metadata is
 ### 3.1 Ossie → OBML
 
 1. Parse `source` string to extract `database`, `schema`, and `table`
-2. Convert fields to columns with type inference (heuristic-based `abstractType`)
+2. Convert fields to columns; `abstractType` comes from the spec `datatype` (`Decimal` narrows to `float`), then legacy `data_type`, then a name heuristic (also used for `Opaque`); metric `datatype` sets the exact measure/metric `dataType` (`Decimal` → the model's `settings.defaultNumericDataType`, else `decimal(18, 2)`). A stashed `obml_abstract_type` or `obml_data_type` is restored while it agrees with `datatype`; an edited `datatype` wins over it
 3. Restructure global relationships into inline joins on data objects
 4. Decompose metric SQL expressions into OBML measures + metrics
 5. Extract dimension-flagged fields into the top-level `dimensions` section (excluding FK/PK join keys)

converters/orionbelt/src/ossie_orionbelt/_common.py

@@ -91,3 +91,124 @@
     "timestamp": "timestamp",
     "boolean": "boolean",
 }
+
+# ─── Ossie DataType ⇄ OBML ──────────────────────────────────────────────────
+# Ossie `datatype` on Field/Metric is a *logical* type backed by the capitalised
+# `DataType` enum in core-spec/ossie-schema.json - the same layer as OBML's column
+# `abstractType` - so this is the field/dimension mapping.
+#
+# `Decimal` has no logical-layer equivalent in OBML: OBML models exact decimal at
+# the physical/result layer (`sqlType`/`sqlPrecision`/`sqlScale`, measure/metric
+# `dataType` via `decimal(p, s)`), not as a coarse `abstractType`. So `Decimal`
+# narrows to `float` for fields, but is recovered exactly for metrics via the
+# physical `dataType` map below (`OSSIE_DATATYPE_TO_OBML_PHYSICAL`).
+#
+# `Opaque` is Ossie's "known type outside the portable vocabulary" marker and is
+# intentionally absent so it falls back to the name heuristic on import.
+OSSIE_DATATYPE_TO_OBML_ABSTRACT = {
+    "String": "string",
+    "Integer": "int",
+    "Float": "float",
+    "Decimal": "float",
+    "Boolean": "boolean",
+    "Date": "date",
+    "Time": "time",
+    "DateTime": "timestamp",
+    "DateTimeTz": "timestamp_tz",
+}
+
+# OBML column `abstractType` -> Ossie `DataType`, for the export direction.
+OBML_ABSTRACT_TO_OSSIE_DATATYPE = {
+    "string": "String",
+    "json": "Opaque",
+    "int": "Integer",
+    "float": "Float",
+    "date": "Date",
+    "time": "Time",
+    "time_tz": "Time",
+    "timestamp": "DateTime",
+    "timestamp_tz": "DateTimeTz",
+    "boolean": "Boolean",
+}
+
+# Metric/measure `datatype`. Unlike fields, OBML measures/metrics carry an exact
+# `dataType` (physical vocabulary: `integer`/`double`/`decimal(p, s)`/...), which
+# is where `Decimal` genuinely belongs. So Ossie metric `datatype` maps to that
+# field, not the coarse `abstractType`.
+OBML_DECIMAL_DEFAULT = "decimal(18, 2)"  # mirrors OrionBelt's built-in default
+
+# Ossie `DataType` -> OBML physical `dataType` (import direction). `Opaque` is
+# omitted (non-portable). `DateTimeTz` has no tz-aware physical form, so it
+# narrows to `timestamp`.
+OSSIE_DATATYPE_TO_OBML_PHYSICAL = {
+    "String": "string",
+    "Integer": "integer",
+    "Float": "double",
+    "Decimal": OBML_DECIMAL_DEFAULT,
+    "Boolean": "boolean",
+    "Date": "date",
+    "Time": "time",
+    "DateTime": "timestamp",
+    "DateTimeTz": "timestamp",
+}
+
+# OBML physical `dataType` -> Ossie `DataType` (export direction). `decimal(p, s)`
+# is handled by ``obml_datatype_to_ossie`` since it is parametrised.
+OBML_PHYSICAL_TO_OSSIE_DATATYPE = {
+    "string": "String",
+    "integer": "Integer",
+    "bigint": "Integer",
+    "double": "Float",
+    "boolean": "Boolean",
+    "date": "Date",
+    "time": "Time",
+    "timestamp": "DateTime",
+}
+
+
+def obml_datatype_to_ossie(data_type: object) -> str | None:
+    """Map an explicit OBML measure/metric ``dataType`` to an Ossie ``DataType``.
+
+    Returns ``None`` when there is no mapping, so the caller emits nothing rather
+    than an unknown type. ``decimal(p, s)`` maps to ``Decimal``. A hand-authored
+    document may carry a non-string ``dataType`` (``123``) that no schema check
+    has rejected yet; that has no mapping either, rather than aborting the whole
+    conversion.
+    """
+    if not isinstance(data_type, str):
+        return None
+    normalized = data_type.strip().lower()
+    if not normalized:
+        return None
+    if normalized.startswith("decimal"):
+        return "Decimal"
+    return OBML_PHYSICAL_TO_OSSIE_DATATYPE.get(normalized)
+
+
+def obml_decimal_default(settings: object) -> str:
+    """The ``dataType`` an Ossie ``Decimal`` metric becomes in this model.
+
+    OBML lets a model set ``settings.defaultNumericDataType`` (always a
+    ``decimal(p, s)``, which OrionBelt enforces), and a model configured for
+    ``decimal(20, 6)`` should not have its metrics written as the built-in
+    ``decimal(18, 2)``. Anything other than a decimal string there falls back to
+    the built-in default.
+    """
+    if isinstance(settings, dict):
+        configured = settings.get("defaultNumericDataType")
+        if isinstance(configured, str) and configured.strip().lower().startswith("decimal"):
+            return configured
+    return OBML_DECIMAL_DEFAULT
+
+
+def ossie_metric_datatype_to_obml(ossie_datatype: object, decimal_default: str) -> str | None:
+    """Map an Ossie metric ``datatype`` to the OBML measure/metric ``dataType``.
+
+    ``Decimal`` takes the model's numeric default; ``Opaque``, an unknown value
+    or a non-string has no mapping.
+    """
+    if not isinstance(ossie_datatype, str):
+        return None
+    if ossie_datatype == "Decimal":
+        return decimal_default
+    return OSSIE_DATATYPE_TO_OBML_PHYSICAL.get(ossie_datatype)

converters/orionbelt/src/ossie_orionbelt/obml_to_ossie.py

@@ -32,7 +32,9 @@
     _OSSIE_VENDOR_READ,
     _OSSIE_VERSION,
     _VENDOR_OBML,
+    OBML_ABSTRACT_TO_OSSIE_DATATYPE,
     OBML_TO_OSSIE_TYPE,
+    obml_datatype_to_ossie,
 )
 
 
@@ -385,8 +387,12 @@ def _convert_column(
         if ai_ctx:
             field["ai_context"] = ai_ctx
 
-        # Preserve OBML type info in custom_extensions for roundtrip fidelity
+        # Emit the spec `datatype` from the OBML abstractType so exported fields
+        # carry a portable logical type...
         abstract_type = col_obj.get("abstractType", "string")
+        field["datatype"] = OBML_ABSTRACT_TO_OSSIE_DATATYPE.get(abstract_type, "String")
+        # ...and stash the exact abstractType in custom_extensions so the return
+        # trip restores it verbatim, lossless through the narrowing map.
         ossie_type = OBML_TO_OSSIE_TYPE.get(abstract_type, "string")
         ext_data: dict[str, Any] = {
             "data_type": ossie_type,
@@ -549,6 +555,7 @@ def _convert_measures_and_metrics(
             ossie_metric = self._convert_measure(measure_name, measure_obj, data_objects)
             if ossie_metric:
                 self._carry_foreign_to_ossie_metric(measure_obj, ossie_metric)
+                self._emit_ossie_metric_datatype(measure_obj, ossie_metric)
                 ossie_metrics.append(ossie_metric)
 
         # Convert OBML metrics (which reference measures) to Ossie metrics
@@ -571,6 +578,7 @@ def _convert_measures_and_metrics(
                 )
             if ossie_metric:
                 self._carry_foreign_to_ossie_metric(metric_obj, ossie_metric)
+                self._emit_ossie_metric_datatype(metric_obj, ossie_metric)
                 ossie_metrics.append(ossie_metric)
 
         return ossie_metrics
@@ -584,6 +592,18 @@ def _carry_foreign_to_ossie_metric(self, obml_obj: dict, ossie_metric: dict) ->
         if not ossie_metric["custom_extensions"]:
             del ossie_metric["custom_extensions"]
 
+    def _emit_ossie_metric_datatype(self, obml_obj: dict, ossie_metric: dict) -> None:
+        """Emit the spec `datatype` from an explicit OBML measure/metric `dataType`.
+
+        Only fires when the OBML object declares an exact `dataType`, so plain
+        measures (whose type is only the defaulted `resultType`) stay untouched
+        and round trips stay idempotent. The exact `dataType` also round-trips via
+        `obml_data_type` in `custom_extensions`; this adds the portable field.
+        """
+        ossie_dt = obml_datatype_to_ossie(obml_obj.get("dataType"))
+        if ossie_dt:
+            ossie_metric["datatype"] = ossie_dt
+
     def _convert_measure(self, name: str, measure: dict, data_objects: dict) -> dict | None:
         """Convert an OBML measure to an Ossie metric."""
 

converters/orionbelt/src/ossie_orionbelt/ossie_to_obml.py

@@ -34,7 +34,13 @@
     _OSSIE_VERSION,
     _SQL_PARSEABLE_DIALECTS,
     _VENDOR_OSSIE,
+    OBML_ABSTRACT_TO_OSSIE_DATATYPE,
+    OBML_DECIMAL_DEFAULT,
+    OSSIE_DATATYPE_TO_OBML_ABSTRACT,
     OSSIE_TO_OBML_TYPE,
+    obml_datatype_to_ossie,
+    obml_decimal_default,
+    ossie_metric_datatype_to_obml,
 )
 
 # A dataset/column identifier in a resolved metric expression: either a bare SQL
@@ -58,6 +64,10 @@ def __init__(
         # or an expression our parser cannot decompose). Preserved verbatim
         # rather than dropped — see ``_preserve_unconverted_metric``.
         self._unconverted_metrics: list[dict] = []
+        # What an Ossie ``Decimal`` metric becomes: the model's own
+        # ``settings.defaultNumericDataType`` when it carries one, set per model
+        # in ``convert``.
+        self._decimal_default = OBML_DECIMAL_DEFAULT
 
     def _normalize_legacy_v01(self) -> None:
         """Promote Ossie v0.1.x payloads to the v0.2 shape, in place.
@@ -168,6 +178,7 @@ def convert(self) -> dict:
 
         # ── Measures & Metrics ──────────────────────────────────────
         ossie_metrics = model.get("metrics", [])
+        self._decimal_default = obml_decimal_default(self._stashed_obml_settings(model))
         measures, metrics = self._convert_metrics(ossie_metrics, ds_map)
         if measures:
             obml["measures"] = measures
@@ -211,6 +222,22 @@ def convert(self) -> dict:
 
         return obml
 
+    @staticmethod
+    def _stashed_obml_settings(model: dict) -> object:
+        """The OBML ``settings`` an OBML-origin model stashed on export, if any.
+
+        Read ahead of the metrics, which need the numeric default; the model
+        properties themselves are restored after them, as before.
+        """
+        for ext in model.get("custom_extensions", []):
+            if ext.get("vendor_name") in _OBML_VENDOR_READ:
+                try:
+                    data = json.loads(ext.get("data", "{}"))
+                except (json.JSONDecodeError, TypeError):
+                    return None
+                return data.get("obml_settings") if isinstance(data, dict) else None
+        return None
+
     @staticmethod
     def _carry_foreign_extensions(ossie_exts: list[dict] | None, obml_target: dict[str, Any]) -> None:
         """Carry third-party Ossie custom_extensions verbatim into OBML.
@@ -374,10 +401,22 @@ def _convert_field(self, field: dict) -> tuple[str, dict]:
             elif code == name and dialects:
                 code = dialects[0].get("expression", name)
 
-        # Determine abstract type: prefer explicit data_type, fall back to heuristic
-        ossie_type = field.get("data_type", "")
-        if ossie_type and ossie_type in OSSIE_TO_OBML_TYPE:
-            abstract_type = OSSIE_TO_OBML_TYPE[ossie_type]
+        # Determine abstract type. Precedence: the spec `datatype` (capitalised
+        # `DataType` enum) > legacy lowercase `data_type` > name heuristic. An
+        # OBML-origin field additionally restores its exact `abstractType` from
+        # the stashed extension below, keeping OBML -> Ossie -> OBML lossless,
+        # unless the `datatype` was edited since. A non-string value (a
+        # hand-authored document) counts as absent rather than crashing.
+        ossie_datatype = field.get("datatype")
+        if not isinstance(ossie_datatype, str):
+            ossie_datatype = ""
+        legacy_type = field.get("data_type")
+        if not isinstance(legacy_type, str):
+            legacy_type = ""
+        if ossie_datatype in OSSIE_DATATYPE_TO_OBML_ABSTRACT:
+            abstract_type = OSSIE_DATATYPE_TO_OBML_ABSTRACT[ossie_datatype]
+        elif legacy_type and legacy_type in OSSIE_TO_OBML_TYPE:
+            abstract_type = OSSIE_TO_OBML_TYPE[legacy_type]
         else:
             abstract_type = self._infer_obml_type(field)
 
@@ -411,6 +450,21 @@ def _convert_field(self, field: dict) -> tuple[str, dict]:
             if ext.get("vendor_name") in _OBML_VENDOR_READ:
                 try:
                     ext_data = json.loads(ext.get("data", "{}"))
+                    # Restore the exact OBML abstractType stashed on export, so a
+                    # narrowing datatype map (e.g. time_tz -> Time) never
+                    # degrades an OBML-origin round trip. The stash yields to a
+                    # `datatype` that no longer agrees with it: that is an edit
+                    # made in Ossie after the export, and it is the newer fact.
+                    stashed = ext_data.get("obml_abstract_type")
+                    if isinstance(stashed, str) and stashed:
+                        stashed_ossie = OBML_ABSTRACT_TO_OSSIE_DATATYPE.get(stashed)
+                        edited = (
+                            ossie_datatype in OSSIE_DATATYPE_TO_OBML_ABSTRACT
+                            and stashed_ossie is not None
+                            and stashed_ossie != ossie_datatype
+                        )
+                        if not edited:
+                            col["abstractType"] = stashed
                     if ext_data.get("obml_sql_type"):
                         col["sqlType"] = ext_data["obml_sql_type"]
                     if ext_data.get("obml_sql_precision") is not None:
@@ -829,6 +883,21 @@ def _convert_metrics(self, ossie_metrics: list, ds_map: dict) -> tuple[dict, dic
             target = metrics.get(m["name"]) or measures.get(m["name"])
             if target is not None:
                 self._carry_foreign_extensions(m.get("custom_extensions"), target)
+                # Ossie metric `datatype` -> OBML exact `dataType` (its natural
+                # home; `Decimal` -> the model's decimal(p, s)). Opaque, unknown
+                # and non-string values have no mapping and change nothing.
+                ossie_dt = m.get("datatype")
+                obml_dt = ossie_metric_datatype_to_obml(ossie_dt, self._decimal_default)
+                if obml_dt is None:
+                    continue
+                # A dataType restored from the OBML-origin stash is more exact
+                # than the map (`decimal(20, 6)`, `bigint`) and is kept while it
+                # still agrees with `datatype`. When it names a different type,
+                # `datatype` was edited in Ossie after the export, and the edit
+                # wins over the stale stash.
+                stashed_ossie = obml_datatype_to_ossie(target.get("dataType"))
+                if stashed_ossie is None or stashed_ossie != ossie_dt:
+                    target["dataType"] = obml_dt
 
         return measures, metrics