Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
659cabe
wip: working inference processor
alfieroddan Jun 29, 2026
ef670f0
wip: changed preprocessor
alfieroddan Jul 2, 2026
edd6052
merge: merge upstream
alfieroddan Jul 11, 2026
1e6c40b
wip: molmo-preprocessors
alfieroddan Jul 12, 2026
d8728f6
Merge branch 'ar/molmoact2' of https://github.com/alfieroddan/physica…
alfieroddan Jul 12, 2026
00a5512
wip: molmo pre and post processors
alfieroddan Aug 10, 2026
c18449e
Merge branch 'openvinotoolkit:main' into ar/molmoact2
alfieroddan Aug 10, 2026
9fafe3f
merge
alfieroddan Aug 10, 2026
a981920
wip: working pre / post processors
alfieroddan Aug 10, 2026
0b8e35d
wip: working pre / post processors
alfieroddan Aug 10, 2026
1789fb9
wip: working asset tokenizer
alfieroddan Aug 10, 2026
6749106
wip: working ov tokenizer
alfieroddan Aug 10, 2026
80bb416
molmo pre / post processors
alfieroddan Aug 11, 2026
1f33726
fix: prek lines
alfieroddan Aug 11, 2026
edc259c
Merge branch 'main' into ar/molmoact2
alfieroddan Aug 12, 2026
6818fc8
prek: fix
alfieroddan Aug 12, 2026
7eb2ffb
restore: restore changed files
alfieroddan Aug 12, 2026
8b747d5
ruff: refactor for ruff
alfieroddan Aug 12, 2026
6bd331d
fix: accept B, H, W, C input
alfieroddan Aug 20, 2026
3a59746
refactor: simplify and slim, seperate into individual fiels
alfieroddan Aug 27, 2026
6fb8758
Merge branch 'main' into ar/molmoact2
alfieroddan Aug 28, 2026
5bd0ac4
fix: copilot comments
alfieroddan Aug 28, 2026
1b9c5de
fix: add token start and end for ov export
alfieroddan Aug 28, 2026
2740fb8
Merge branch 'main' into ar/molmoact2
alfieroddan Sep 2, 2026
2dddfc7
Merge branch 'openvinotoolkit:main' into ar/molmoact2
alfieroddan Sep 3, 2026
68bed07
feat: joint frame transformation processor, factor out for molmo
alfieroddan Sep 9, 2026
868ea11
refactor: forward and inverse joint transformations
alfieroddan Sep 9, 2026
628cf1c
refactor: move joint_transform to postprocess
alfieroddan Sep 11, 2026
5ab8154
feat: add action key to postprocessor
alfieroddan Sep 11, 2026
6231b2b
Merge branch 'openvinotoolkit:main' into ar/molmoact2
alfieroddan Sep 15, 2026
176df79
Merge branch 'openvinotoolkit:main' into ar/molmoact2
alfieroddan Sep 15, 2026
bcac132
refactor: move molmo processors to own subfolder
alfieroddan Sep 16, 2026
808818f
Merge branch 'main' into ar/molmoact2
alfieroddan Sep 16, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions src/physicalai/inference/component_factory.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,16 +108,22 @@ def __repr__(self) -> str:
component_registry.register("smolvla_resize", "physicalai.inference.preprocessors.ResizeSmolVLA")
component_registry.register("new_line", "physicalai.inference.preprocessors.NewLinePreprocessor")
component_registry.register("hf_tokenizer", "physicalai.inference.preprocessors.HFTokenizer")
component_registry.register("molmoact2", "physicalai.inference.preprocessors.MolmoAct2Preprocessor")
component_registry.register("molmoact2_inputs", "physicalai.inference.preprocessors.MolmoAct2ModelInputs")
component_registry.register("ov_tokenizer", "physicalai.inference.preprocessors.OVTokenizer")
component_registry.register("pi05", "physicalai.inference.preprocessors.Pi05Preprocessor")
component_registry.register("rldx1", "physicalai.inference.preprocessors.Rldx1Preprocessor")
component_registry.register("rldx1_token_composer", "physicalai.inference.preprocessors.Rldx1TokenComposer")
component_registry.register("rldx1_rope", "physicalai.inference.preprocessors.Rldx1RopePreprocessor")
component_registry.register("to_float_tensor", "physicalai.inference.preprocessors.ToFloatTensorPreprocessor")
component_registry.register("molmoact2_pre", "physicalai.inference.preprocessors.MolmoAct2Preprocessor")
component_registry.register("joint_frame_preprocess", "physicalai.inference.preprocessors.JointFramePreprocessor")

# Postprocessors
component_registry.register("denormalize", "physicalai.inference.postprocessors.StatsDenormalizer")
component_registry.register("action_chunk_trimmer", "physicalai.inference.postprocessors.ActionChunkTrimmer")
component_registry.register("molmoact2_postprocess", "physicalai.inference.postprocessors.MolmoAct2Postprocessor")
component_registry.register("joint_frame_postprocess", "physicalai.inference.postprocessors.JointFramePostprocessor")

# Callbacks
component_registry.register("latency_monitor", "physicalai.inference.callbacks.LatencyMonitor")
Expand Down
4 changes: 4 additions & 0 deletions src/physicalai/inference/postprocessors/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,11 +9,15 @@
from physicalai.inference.postprocessors.action_chunk_trimmer import ActionChunkTrimmer
from physicalai.inference.postprocessors.action_normalizer import ActionNormalizer
from physicalai.inference.postprocessors.base import Postprocessor
from physicalai.inference.postprocessors.joint_frame import JointFramePostprocessor
from physicalai.inference.postprocessors.molmoact2 import MolmoAct2Postprocessor
from physicalai.inference.postprocessors.stats_denormalizer import StatsDenormalizer

__all__ = [
"ActionChunkTrimmer",
"ActionNormalizer",
"JointFramePostprocessor",
"MolmoAct2Postprocessor",
"Postprocessor",
"StatsDenormalizer",
]
91 changes: 91 additions & 0 deletions src/physicalai/inference/postprocessors/joint_frame.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""Joint-frame action postprocessing."""

from __future__ import annotations

from typing import TYPE_CHECKING

import numpy as np
from typing_extensions import override

from physicalai.inference.postprocessors.base import Postprocessor

if TYPE_CHECKING:
from collections.abc import Sequence


class JointFrameTransform:
"""Apply an invertible affine transform to leading joint values."""

def __init__(self, *, signs: Sequence[float], offsets: Sequence[float]) -> None:
"""Store the joint signs and offsets.

Raises:
ValueError: If signs and offsets differ in length or a sign is not +/-1.
"""
if len(signs) != len(offsets):
msg = f"signs ({len(signs)}) and offsets ({len(offsets)}) must match"
raise ValueError(msg)
if any(sign not in {-1.0, 1.0} for sign in signs):
msg = "Joint frame transform signs must be either -1 or 1."
raise ValueError(msg)
self._signs = np.asarray(signs, dtype=np.float32)
self._offsets = np.asarray(offsets, dtype=np.float32)

def forward(self, values: np.ndarray) -> np.ndarray:
"""Apply ``sign * value + offset`` to leading joint values.

Returns:
A transformed copy of ``values``.
"""
return self._apply(values, inverse=False)

def inverse(self, values: np.ndarray) -> np.ndarray:
"""Apply ``sign * (value - offset)`` to leading joint values.

Returns:
An inverse-transformed copy of ``values``.
"""
return self._apply(values, inverse=True)

def _apply(self, values: np.ndarray, *, inverse: bool) -> np.ndarray:
count = min(self._signs.size, values.shape[-1])
output = np.array(values, copy=True)
joints = values[..., :count]
output[..., :count] = (
self._signs[:count] * (joints - self._offsets[:count])
if inverse
else self._signs[:count] * joints + self._offsets[:count]
)
return output


class JointFramePostprocessor(Postprocessor):
"""Map one output feature from checkpoint to robot joint coordinates."""

def __init__(self, *, feature: str, signs: Sequence[float], offsets: Sequence[float]) -> None:
"""Configure the feature and calibration frame."""
self._feature = feature
self._transform = JointFrameTransform(signs=signs, offsets=offsets)

@override
def __call__(self, outputs: dict[str, np.ndarray]) -> dict[str, np.ndarray]:
"""Transform the configured feature while preserving all other outputs.

Returns:
A shallow copy with the transformed feature.

Raises:
ValueError: If the configured feature is absent.
"""
if self._feature not in outputs:
msg = f"Joint frame postprocessor expected feature {self._feature!r}"
raise ValueError(msg)
result = dict(outputs)
result[self._feature] = self._transform.inverse(np.asarray(outputs[self._feature]))
return result


__all__ = ["JointFramePostprocessor"]
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""MolmoAct2 inference postprocessors."""

from physicalai.inference.postprocessors.molmoact2.processor import MolmoAct2Postprocessor

__all__ = ["MolmoAct2Postprocessor"]
69 changes: 69 additions & 0 deletions src/physicalai/inference/postprocessors/molmoact2/processor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""NumPy postprocessing for MolmoAct2 inference."""

from __future__ import annotations

from typing import Any

import numpy as np
from typing_extensions import override

from physicalai.inference.constants import ACTION
from physicalai.inference.postprocessors.base import Postprocessor
from physicalai.inference.postprocessors.stats_denormalizer import StatsDenormalizer
from physicalai.inference.preprocessors.molmoact2 import normalization_stats


class MolmoAct2Postprocessor(Postprocessor):
"""Clamp and denormalize MolmoAct2 actions."""

def __init__(
self,
*,
action_key: str,
action_stats: dict[str, Any] | None = None,
normalization_mode: str = "QUANTILES",
) -> None:
"""Store action postprocessing settings.

Args:
action_key: Adapter output key containing the action tensor.
action_stats: Statistics used to denormalize actions.
normalization_mode: Normalization strategy used during training.
"""
self._action_key = action_key
self.denormalizer = (
StatsDenormalizer(
stats={ACTION: normalization_stats(action_stats)},
mode=normalization_mode.lower(),
features=[ACTION],
)
if action_stats
else None
)

@override
def __call__(self, outputs: dict[str, np.ndarray]) -> dict[str, np.ndarray]:
"""Postprocess the model action output.

Returns:
Outputs with the canonical denormalized action.

Raises:
ValueError: If the configured action output is absent.
"""
result = dict(outputs)
if self._action_key not in result:
msg = f"MolmoAct2 postprocessor expected action key {self._action_key!r}"
raise ValueError(msg)
action = result.pop(self._action_key)
action = np.clip(np.asarray(action), -1.0, 1.0)
if self.denormalizer is not None:
action = self.denormalizer({ACTION: action})[ACTION]
result[ACTION] = action
return result


__all__ = ["MolmoAct2Postprocessor"]
14 changes: 8 additions & 6 deletions src/physicalai/inference/postprocessors/stats_denormalizer.py
Original file line number Diff line number Diff line change
Expand Up @@ -186,21 +186,23 @@ def _denormalize(
Returns:
Denormalized array.
"""
transformed = tensor
if mode == "mean_std":
mean = stats["mean"]
std = stats["std"]
return tensor * std + mean
transformed = tensor * std + mean

if mode == "min_max":
elif mode == "min_max":
min_val = stats["min"]
max_val = stats["max"]
return (tensor + 1.0) / 2.0 * (max_val - min_val) + min_val
transformed = (tensor + 1.0) / 2.0 * (max_val - min_val) + min_val

if mode == "quantiles":
elif mode == "quantiles":
q01 = stats["q01"]
q99 = stats["q99"]
denom = q99 - q01
denom = np.where(denom == 0, _EPS, denom)
return (tensor + 1.0) * denom / 2.0 + q01
transformed = (tensor + 1.0) * denom / 2.0 + q01

return tensor
mask = stats.get("mask")
return np.where(mask.astype(np.bool_), transformed, tensor) if mask is not None else transformed
5 changes: 5 additions & 0 deletions src/physicalai/inference/preprocessors/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,9 @@

from physicalai.inference.preprocessors.base import Preprocessor
from physicalai.inference.preprocessors.hf_tokenizer import HFTokenizer
from physicalai.inference.preprocessors.joint_frame import JointFramePreprocessor
from physicalai.inference.preprocessors.lambda_processor import LambdaPreprocessor
from physicalai.inference.preprocessors.molmoact2 import MolmoAct2ModelInputs, MolmoAct2Preprocessor
from physicalai.inference.preprocessors.new_line import NewLinePreprocessor
from physicalai.inference.preprocessors.ov_tokenizer import OVTokenizer
from physicalai.inference.preprocessors.pi05 import Pi05Preprocessor
Expand All @@ -23,7 +25,10 @@

__all__ = [
"HFTokenizer",
"JointFramePreprocessor",
"LambdaPreprocessor",
"MolmoAct2ModelInputs",
"MolmoAct2Preprocessor",
"NewLinePreprocessor",
"OVTokenizer",
"Pi05Preprocessor",
Expand Down
50 changes: 50 additions & 0 deletions src/physicalai/inference/preprocessors/joint_frame.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,50 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""Joint-frame observation preprocessing."""

from __future__ import annotations

from typing import TYPE_CHECKING, Any

import numpy as np
from typing_extensions import override

from physicalai.inference.postprocessors.joint_frame import JointFrameTransform
from physicalai.inference.preprocessors.base import Preprocessor

if TYPE_CHECKING:
from collections.abc import Sequence


class JointFramePreprocessor(Preprocessor):
"""Map one observation feature from robot to checkpoint joint coordinates."""

def __init__(self, *, feature: str, signs: Sequence[float], offsets: Sequence[float]) -> None:
"""Configure the feature and calibration frame."""
self._feature = feature
self._transform = JointFrameTransform(signs=signs, offsets=offsets)

@override
def __call__(self, inputs: dict[str, Any]) -> dict[str, Any]:
"""Transform the configured feature while preserving all other inputs.

Returns:
A shallow copy with the transformed feature.
"""
key = self._resolve_key(inputs)
outputs = dict(inputs)
outputs[key] = self._transform.forward(np.asarray(inputs[key]))
return outputs

def _resolve_key(self, inputs: dict[str, Any]) -> str:
if self._feature in inputs:
return self._feature
observation_key = f"observation.{self._feature}"
if observation_key in inputs:
return observation_key
msg = f"Joint frame preprocessor expected feature {self._feature!r}"
raise ValueError(msg)


__all__ = ["JointFramePreprocessor"]
13 changes: 13 additions & 0 deletions src/physicalai/inference/preprocessors/molmoact2/__init__.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# Copyright (C) 2026 Intel Corporation
# SPDX-License-Identifier: Apache-2.0

"""MolmoAct2 inference preprocessors."""

from physicalai.inference.preprocessors.molmoact2.inputs import MolmoAct2ModelInputs
from physicalai.inference.preprocessors.molmoact2.processor import MolmoAct2Preprocessor, normalization_stats

__all__ = [
"MolmoAct2ModelInputs",
"MolmoAct2Preprocessor",
"normalization_stats",
]
Loading
Loading