diff --git a/examples/recipes/MIT_ast-finetuned-audioset-10-10-0.4593/cpu/cpu/audio-classification_fp16_config.json b/examples/recipes/MIT_ast-finetuned-audioset-10-10-0.4593/cpu/cpu/audio-classification_fp16_config.json index 6ac10fb42..723f155bf 100644 --- a/examples/recipes/MIT_ast-finetuned-audioset-10-10-0.4593/cpu/cpu/audio-classification_fp16_config.json +++ b/examples/recipes/MIT_ast-finetuned-audioset-10-10-0.4593/cpu/cpu/audio-classification_fp16_config.json @@ -28,7 +28,10 @@ { "name": "logits" } - ] + ], + "compatibility": { + "transformers_attention": "eager" + } }, "optim": {}, "quant": { diff --git a/examples/recipes/MIT_ast-finetuned-audioset-10-10-0.4593/cpu/cpu/audio-classification_fp32_config.json b/examples/recipes/MIT_ast-finetuned-audioset-10-10-0.4593/cpu/cpu/audio-classification_fp32_config.json index 439f9edee..62aca4526 100644 --- a/examples/recipes/MIT_ast-finetuned-audioset-10-10-0.4593/cpu/cpu/audio-classification_fp32_config.json +++ b/examples/recipes/MIT_ast-finetuned-audioset-10-10-0.4593/cpu/cpu/audio-classification_fp32_config.json @@ -28,7 +28,10 @@ { "name": "logits" } - ] + ], + "compatibility": { + "transformers_attention": "eager" + } }, "optim": {}, "quant": null, diff --git a/examples/recipes/prithivMLmods_Common-Voice-Gender-Detection/cpu/cpu/audio-classification_fp16_config.json b/examples/recipes/prithivMLmods_Common-Voice-Gender-Detection/cpu/cpu/audio-classification_fp16_config.json index c7cd884c5..d466add82 100644 --- a/examples/recipes/prithivMLmods_Common-Voice-Gender-Detection/cpu/cpu/audio-classification_fp16_config.json +++ b/examples/recipes/prithivMLmods_Common-Voice-Gender-Detection/cpu/cpu/audio-classification_fp16_config.json @@ -27,7 +27,10 @@ { "name": "logits" } - ] + ], + "compatibility": { + "transformers_attention": "eager" + } }, "optim": {}, "quant": { diff --git a/examples/recipes/prithivMLmods_Common-Voice-Gender-Detection/cpu/cpu/audio-classification_fp32_config.json b/examples/recipes/prithivMLmods_Common-Voice-Gender-Detection/cpu/cpu/audio-classification_fp32_config.json index 1dce00fb3..bad088b73 100644 --- a/examples/recipes/prithivMLmods_Common-Voice-Gender-Detection/cpu/cpu/audio-classification_fp32_config.json +++ b/examples/recipes/prithivMLmods_Common-Voice-Gender-Detection/cpu/cpu/audio-classification_fp32_config.json @@ -27,7 +27,10 @@ { "name": "logits" } - ] + ], + "compatibility": { + "transformers_attention": "eager" + } }, "optim": {}, "quant": null, diff --git a/src/winml/modelkit/eval/__init__.py b/src/winml/modelkit/eval/__init__.py index 435601a15..cf7084e91 100644 --- a/src/winml/modelkit/eval/__init__.py +++ b/src/winml/modelkit/eval/__init__.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: + from .audio_classification_evaluator import WinMLAudioClassificationEvaluator from .depth_estimation_evaluator import WinMLDepthEstimationEvaluator from .feature_extraction_evaluator import WinMLFeatureExtractionEvaluator from .fill_mask_evaluator import WinMLFillMaskEvaluator @@ -47,6 +48,9 @@ _LAZY_ATTRS: dict[str, str] = { # Evaluators + "WinMLAudioClassificationEvaluator": ( + ".audio_classification_evaluator:WinMLAudioClassificationEvaluator" + ), "WinMLDepthEstimationEvaluator": ".depth_estimation_evaluator:WinMLDepthEstimationEvaluator", "WinMLFeatureExtractionEvaluator": ( ".feature_extraction_evaluator:WinMLFeatureExtractionEvaluator" @@ -126,6 +130,7 @@ def __dir__() -> list[str]: "SpearmanCorrelationMetric", "TensorSimilarityEvaluator", "TopKAccuracyMetric", + "WinMLAudioClassificationEvaluator", "WinMLDepthEstimationEvaluator", "WinMLEvaluationConfig", "WinMLEvaluator", diff --git a/src/winml/modelkit/eval/audio_classification_evaluator.py b/src/winml/modelkit/eval/audio_classification_evaluator.py new file mode 100644 index 000000000..8bb24c480 --- /dev/null +++ b/src/winml/modelkit/eval/audio_classification_evaluator.py @@ -0,0 +1,480 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +"""Audio classification evaluation for scalar and multi-label targets.""" + +from __future__ import annotations + +import math +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any, cast + +import numpy as np + +from ..utils.eval_utils import DatasetValidationError +from .base_evaluator import WinMLEvaluator + + +if TYPE_CHECKING: + from datasets import Dataset + from numpy.typing import NDArray + from transformers.pipelines.base import Pipeline + + from .config import DatasetConfig, WinMLEvaluationConfig + + +class _AudioModelAdapter: + """One preprocessing and forward contract for native-HF and WinML models.""" + + def __init__(self, config: WinMLEvaluationConfig, model: Any) -> None: + from transformers import AutoFeatureExtractor + + if not config.model_id: + raise ValueError("model_id is required to load the audio feature extractor.") + self._config = config + self._model = model + self._feature_extractor = AutoFeatureExtractor.from_pretrained( + config.model_id, + trust_remote_code=config.trust_remote_code, + ) + self._input_contract = self._resolve_input_contract() + + def predict_logits(self, audio: Any) -> NDArray[np.float32]: + """Decode and process one audio row, then perform exactly one forward.""" + import torch + + waveform, sampling_rate = self._decode_audio(audio) + waveform = self._downmix(waveform) + target_rate = int(getattr(self._feature_extractor, "sampling_rate", sampling_rate)) + if sampling_rate != target_rate: + waveform = self._resample(waveform, sampling_rate, target_rate) + + processor_kwargs: dict[str, Any] = { + "sampling_rate": target_rate, + "return_tensors": "pt", + } + if self._input_contract is not None and len(self._input_contract[1]) == 2: + sequence_length = self._input_contract[1][1] + processor_kwargs.update( + padding="max_length", + truncation=True, + max_length=sequence_length, + ) + encoded = self._feature_extractor(waveform, **processor_kwargs) + model_inputs = self._select_model_inputs(encoded) + device = self._config.pipeline_device if self._config.runtime == "pytorch" else "cpu" + model_inputs = { + name: ( + value.to(device) + if hasattr(value, "to") + else torch.as_tensor(value, device=device) + ) + for name, value in model_inputs.items() + } + outputs = self._model(**model_inputs) + logits = self._extract_logits(outputs) + array = logits.detach().float().cpu().numpy() if hasattr(logits, "detach") else logits + array = np.asarray(array, dtype=np.float32) + if array.ndim != 2 or array.shape[0] != 1: + raise ValueError(f"expected logits shape [1, classes], got {array.shape}") + return cast("NDArray[np.float32]", array[0]) + + def _resolve_input_contract(self) -> tuple[str, list[int]] | None: + io_config = getattr(self._model, "io_config", None) or {} + names = io_config.get("input_names") or [] + shapes = io_config.get("input_shapes") or [] + if not names and not shapes: + return None + if len(names) != 1 or len(shapes) != 1: + raise ValueError( + "audio-classification adapter supports one model input; " + f"got names={names}, shapes={shapes}" + ) + shape = list(shapes[0]) + if len(shape) < 2 or any(not isinstance(value, int) for value in shape[1:]): + raise ValueError( + "audio-classification adapter requires static non-batch input dimensions; " + f"got {shape}" + ) + if isinstance(shape[0], int) and shape[0] != 1: + raise ValueError("audio-classification adapter requires batch size 1.") + return str(names[0]), [1, *(int(value) for value in shape[1:])] + + def _select_model_inputs(self, encoded: Any) -> dict[str, Any]: + values = dict(encoded) + if self._input_contract is None: + if not values: + raise ValueError("audio feature extractor produced no model inputs.") + return values + + input_name, expected_shape = self._input_contract + if input_name not in values: + raise ValueError( + f"audio feature extractor output must contain {input_name!r}; " + f"got {sorted(values)}" + ) + tensor = values[input_name] + actual_shape = list(getattr(tensor, "shape", ())) + if actual_shape != expected_shape: + raise ValueError( + f"audio feature extractor produced {input_name} shape {actual_shape}; " + f"expected {expected_shape}" + ) + return {input_name: tensor} + + @staticmethod + def _extract_logits(outputs: Any) -> Any: + logits = ( + outputs.get("logits") + if isinstance(outputs, dict) + else getattr(outputs, "logits", None) + ) + if logits is None: + raise ValueError("audio-classification model output does not contain logits.") + return logits + + @staticmethod + def _decode_audio(audio: Any) -> tuple[np.ndarray, int]: + if isinstance(audio, dict): + if audio.get("array") is not None and audio.get("sampling_rate") is not None: + return np.asarray(audio["array"], dtype=np.float32), int(audio["sampling_rate"]) + encoded_bytes = audio.get("bytes") + encoded_path = audio.get("path") + if encoded_bytes is not None or encoded_path: + try: + import soundfile as sf + except ImportError as error: + raise RuntimeError( + "Encoded audio decoding requires the optional audio dependencies." + ) from error + source = BytesIO(encoded_bytes) if encoded_bytes is not None else str(encoded_path) + try: + waveform, sampling_rate = sf.read( + source, + dtype="float32", + always_2d=False, + ) + except (OSError, RuntimeError) as error: + raise ValueError(f"failed to decode audio: {error}") from error + return np.asarray(waveform, dtype=np.float32), int(sampling_rate) + raise TypeError( + "audio value must contain array and sampling_rate, or encoded bytes/path" + ) + + @staticmethod + def _downmix(waveform: np.ndarray) -> np.ndarray: + waveform = np.asarray(waveform, dtype=np.float32) + if waveform.ndim == 1: + mono = waveform + elif waveform.ndim == 2: + mono = waveform.mean(axis=1) + else: + raise ValueError(f"audio must be mono or frames-by-channels, got {waveform.shape}") + if mono.size == 0: + raise ValueError("audio waveform is empty.") + return np.asarray(mono, dtype=np.float32) + + @staticmethod + def _resample(waveform: np.ndarray, source_rate: int, target_rate: int) -> np.ndarray: + from scipy.signal import resample_poly + + if source_rate <= 0 or target_rate <= 0: + raise ValueError("audio sampling rates must be positive.") + divisor = math.gcd(source_rate, target_rate) + return np.asarray( + resample_poly(waveform, target_rate // divisor, source_rate // divisor), + dtype=np.float32, + ) + + +class WinMLAudioClassificationEvaluator(WinMLEvaluator): + """Evaluate audio classifiers with schema-driven target semantics.""" + + def __init__(self, config: WinMLEvaluationConfig, model: Any) -> None: + mapping = config.dataset.columns_mapping + self._audio_column = mapping.get("input_column", "audio") + self._label_column = mapping.get("label_column", "label") + self._label_name_column = mapping.get("label_name_column", "human_labels") + self._target_kind = "" + self._label_feature: Any = None + self._model_id2label, self._model_label2id = self._model_labels(model) + super().__init__(config, model) + + def prepare_pipeline(self) -> Pipeline: + """Create the shared native-HF/WinML audio adapter.""" + return cast("Pipeline", _AudioModelAdapter(self.config, self.model)) + + def prepare_data(self) -> Dataset: + """Load, validate, explicitly disable media decoding, and select rows.""" + from datasets import ( + Audio, + Dataset, + DatasetDict, + IterableDataset, + load_dataset, + load_from_disk, + ) + + ds = self.config.dataset + try: + ds_path = Path(ds.path).expanduser() if ds.path else None + if ds_path and ds_path.is_dir(): + loaded = load_from_disk(str(ds_path)) + if isinstance(loaded, DatasetDict): + if ds.split not in loaded: + raise DatasetValidationError( + f"saved dataset has no split {ds.split!r}; available: {sorted(loaded)}" + ) + dataset = loaded[ds.split] + else: + dataset = loaded + else: + dataset = load_dataset( + ds.path, + name=ds.name, + split=ds.split, + streaming=ds.streaming, + revision=ds.revision, + ) + except DatasetValidationError: + raise + except Exception as error: + raise DatasetValidationError( + f"Failed to load dataset {ds.path!r} (name={ds.name!r}, split={ds.split!r}): " + f"{error}" + ) from error + + self._validate_target_schema(dataset) + audio_feature = dataset.features[self._audio_column] + if isinstance(audio_feature, Audio) and audio_feature.decode: + dataset = dataset.cast_column( + self._audio_column, + Audio(sampling_rate=audio_feature.sampling_rate, decode=False), + ) + if ds.shuffle: + dataset = dataset.shuffle(seed=ds.seed) + if isinstance(dataset, IterableDataset): + dataset = Dataset.from_list(list(dataset.take(ds.samples))) + else: + dataset = dataset.select(range(min(ds.samples, len(dataset)))) + return dataset + + def align_labels(self, dataset: Dataset, ds_config: DatasetConfig) -> Dataset: + """Leave target alignment to the scalar/multi-label decoder.""" + return dataset + + def compute(self) -> dict[str, Any]: + """Run one forward per selected row and compute task-appropriate metrics.""" + logits: list[np.ndarray] = [] + selected_targets = [self._target_for_row(row) for row in self.data] + processed_targets: list[Any] = [] + rejected_by_reason: dict[str, int] = {} + adapter = cast("_AudioModelAdapter", self.pipe) + for row, target in zip(self.data, selected_targets, strict=True): + try: + prediction = adapter.predict_logits(row[self._audio_column]) + except (TypeError, ValueError, RuntimeError) as error: + reason = type(error).__name__ + rejected_by_reason[reason] = rejected_by_reason.get(reason, 0) + 1 + continue + logits.append(prediction) + processed_targets.append(target) + if not logits: + raise DatasetValidationError("No selected audio samples were successfully processed.") + scores = np.stack(logits) + if scores.shape[1] != len(self._model_id2label): + raise DatasetValidationError( + f"model returned {scores.shape[1]} classes but config defines " + f"{len(self._model_id2label)} labels." + ) + if self._target_kind == "multi-label": + metrics = self._multi_label_metrics(scores, processed_targets) + else: + metrics = self._single_label_metrics(scores, processed_targets) + selected = len(self.data) + metrics.update( + requested_samples=self.config.dataset.samples, + selected_samples=selected, + processed_samples=len(processed_targets), + rejected_samples=selected - len(processed_targets), + rejected_by_reason=dict(sorted(rejected_by_reason.items())), + ) + return metrics + + def _validate_target_schema(self, dataset: Any) -> None: + from datasets import ClassLabel, Sequence, Value + + missing = [ + name + for name in (self._audio_column, self._label_column) + if name not in dataset.column_names + ] + if missing: + raise DatasetValidationError( + f"missing required column(s) {missing}; dataset has {sorted(dataset.column_names)}" + ) + feature = dataset.features[self._label_column] + if isinstance(feature, ClassLabel): + self._target_kind = "single-label" + elif isinstance(feature, Value) and feature.dtype == "string": + self._validate_scalar_string_mapping() + self._target_kind = "single-label-string" + elif isinstance(feature, Sequence) and isinstance(feature.feature, (ClassLabel, Value)): + self._target_kind = "multi-label" + else: + raise DatasetValidationError( + f"Column {self._label_column!r} must be ClassLabel or a sequence of " + "ClassLabel/string values, or a scalar string with an explicit label mapping; " + f"got {feature!r}." + ) + self._label_feature = feature + + def _target_for_row(self, row: dict[str, Any]) -> Any: + from datasets import ClassLabel + + raw_target = row[self._label_column] + if self._target_kind == "single-label": + assert isinstance(self._label_feature, ClassLabel) + return self._resolve_label(self._label_feature.int2str(int(raw_target))) + if self._target_kind == "single-label-string": + mapping = self.config.dataset.label_mapping + assert mapping is not None + value = str(raw_target) + if value not in mapping: + raise DatasetValidationError( + f"Dataset label {value!r} is absent from the explicit label mapping." + ) + return int(mapping[value]) + + values = list(raw_target) + feature = self._label_feature.feature + decoded = ( + [feature.int2str(int(value)) for value in values] + if isinstance(feature, ClassLabel) + else values + ) + parallel_names = row.get(self._label_name_column) + if parallel_names is not None and len(parallel_names) != len(decoded): + raise DatasetValidationError( + f"Columns {self._label_column!r} and {self._label_name_column!r} " + "must contain the same number of labels." + ) + resolved: list[int] = [] + for index, value in enumerate(decoded): + fallback_name = parallel_names[index] if parallel_names is not None else None + resolved.append(self._resolve_label(str(value), fallback_name=fallback_name)) + if not resolved: + raise DatasetValidationError("Multi-label targets must contain at least one label.") + return sorted(set(resolved)) + + def _validate_scalar_string_mapping(self) -> None: + mapping = self.config.dataset.label_mapping + if not mapping: + raise DatasetValidationError( + "Scalar string targets require a non-empty explicit label mapping." + ) + if any(not isinstance(label, str) for label in mapping): + raise DatasetValidationError( + "Scalar string label mapping keys must be exact strings." + ) + invalid_destinations = any( + not isinstance(model_id, int) or isinstance(model_id, bool) + for model_id in mapping.values() + ) + if invalid_destinations: + raise DatasetValidationError( + "Scalar string label mapping destinations must be checkpoint IDs." + ) + destinations = list(mapping.values()) + if len(set(destinations)) != len(destinations): + raise DatasetValidationError( + "Scalar string label mapping contains duplicate checkpoint ID destinations." + ) + checkpoint_ids = set(self._model_id2label) + mapped_ids = set(destinations) + unknown_ids = sorted(mapped_ids - checkpoint_ids) + if unknown_ids: + raise DatasetValidationError( + f"Label mapping target IDs {unknown_ids} are absent from model.config.id2label." + ) + missing_ids = sorted(checkpoint_ids - mapped_ids) + if missing_ids: + raise DatasetValidationError( + "Scalar string label mapping must cover every checkpoint ID; " + f"missing {missing_ids}." + ) + + def _resolve_label(self, value: str, *, fallback_name: Any = None) -> int: + mapping = self.config.dataset.label_mapping or {} + if value in mapping: + model_id = int(mapping[value]) + elif value in self._model_label2id: + model_id = self._model_label2id[value] + elif fallback_name is not None and str(fallback_name) in self._model_label2id: + model_id = self._model_label2id[str(fallback_name)] + else: + raise DatasetValidationError( + f"Dataset label {value!r} has no exact model-label match; provide an " + "authoritative label mapping or parallel exact label-name column." + ) + if model_id not in self._model_id2label: + raise DatasetValidationError( + f"Label mapping target {model_id} is absent from model.config.id2label." + ) + return model_id + + def _single_label_metrics( + self, + logits: np.ndarray, + targets: list[int], + ) -> dict[str, Any]: + from .metrics import ClassificationMetric + + predictions = [self._model_id2label[int(index)] for index in np.argmax(logits, axis=1)] + references = [self._model_id2label[int(index)] for index in targets] + represented = sorted(set(references)) + result = ClassificationMetric().compute(predictions, references, represented) + return { + "accuracy": result["accuracy"], + "macro_f1": result["f1"], + "represented_classes": len(represented), + "total_classes": len(self._model_id2label), + "class_coverage": len(represented) / len(self._model_id2label), + } + + def _multi_label_metrics( + self, + logits: np.ndarray, + targets: list[list[int]], + ) -> dict[str, Any]: + from sklearn.metrics import average_precision_score + + references = np.zeros_like(logits, dtype=np.int8) + for row_index, model_ids in enumerate(targets): + references[row_index, model_ids] = 1 + probabilities = 1.0 / (1.0 + np.exp(-logits)) + sample_ap = float(average_precision_score(references, probabilities, average="samples")) + micro_ap = float(average_precision_score(references, probabilities, average="micro")) + if not np.isfinite(sample_ap) or not np.isfinite(micro_ap): + raise DatasetValidationError("Multi-label average precision must be finite.") + return { + "sample_average_precision": sample_ap, + "micro_average_precision": micro_ap, + } + + @staticmethod + def _model_labels(model: Any) -> tuple[dict[int, str], dict[str, int]]: + config = getattr(model, "config", None) + raw_id2label = getattr(config, "id2label", None) or {} + id2label = {int(index): str(label) for index, label in raw_id2label.items()} + if not id2label or sorted(id2label) != list(range(len(id2label))): + raise DatasetValidationError( + "model.config.id2label must define contiguous class IDs starting at zero." + ) + label2id = {label: index for index, label in id2label.items()} + if len(label2id) != len(id2label): + raise DatasetValidationError("model.config.id2label contains duplicate label names.") + return id2label, label2id diff --git a/src/winml/modelkit/eval/evaluate.py b/src/winml/modelkit/eval/evaluate.py index e02ac79a0..548bdd089 100644 --- a/src/winml/modelkit/eval/evaluate.py +++ b/src/winml/modelkit/eval/evaluate.py @@ -60,6 +60,8 @@ def _select_model_loader(config: WinMLEvaluationConfig) -> _ModelLoaderKind: # default formatter layout) yields >100-char lines that trip E501. # fmt: off _EVALUATOR_REGISTRY: dict[str, str] = { + "audio-classification": + "winml.modelkit.eval.audio_classification_evaluator:WinMLAudioClassificationEvaluator", "image-classification": "winml.modelkit.eval.base_evaluator:WinMLEvaluator", "text-classification": diff --git a/src/winml/modelkit/utils/eval_utils.py b/src/winml/modelkit/utils/eval_utils.py index f85c672b4..50b78a400 100644 --- a/src/winml/modelkit/utils/eval_utils.py +++ b/src/winml/modelkit/utils/eval_utils.py @@ -57,6 +57,30 @@ class TaskSchema: ), ) +_AUDIO_CLASSIFICATION_SCHEMA = TaskSchema( + columns=( + SchemaItem( + "input_column", + "encoded audio bytes/path or decoded waveform with sampling rate", + default="audio", + remap_hint="", + ), + SchemaItem( + "label_column", + "scalar ClassLabel, explicitly mapped scalar string, or sequence of exact class labels", + default="label", + remap_hint="", + ), + ), + params=( + SchemaItem( + "label_name_column", + "parallel exact label names for sequence-valued identifiers (optional)", + remap_hint="", + ), + ), +) + _TEXT_CLASSIFICATION_SCHEMA = TaskSchema( columns=( SchemaItem( @@ -449,6 +473,7 @@ class TaskSchema: ) TASK_SCHEMAS: dict[str, TaskSchema] = { + "audio-classification": _AUDIO_CLASSIFICATION_SCHEMA, "image-classification": _IMAGE_CLASSIFICATION_SCHEMA, "text-classification": _TEXT_CLASSIFICATION_SCHEMA, "sequence-classification": _TEXT_CLASSIFICATION_SCHEMA, diff --git a/tests/unit/eval/test_audio_classification_evaluator.py b/tests/unit/eval/test_audio_classification_evaluator.py new file mode 100644 index 000000000..3225a2e99 --- /dev/null +++ b/tests/unit/eval/test_audio_classification_evaluator.py @@ -0,0 +1,596 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +from __future__ import annotations + +from io import BytesIO +from types import SimpleNamespace +from typing import ClassVar +from unittest.mock import patch + +import numpy as np +import pytest +import soundfile as sf +import torch +from datasets import ( + Audio, + ClassLabel, + Dataset, + DatasetDict, + Features, + IterableDataset, + Sequence, + Value, +) + +from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig +from winml.modelkit.eval.audio_classification_evaluator import ( + WinMLAudioClassificationEvaluator, + _AudioModelAdapter, +) +from winml.modelkit.utils.eval_utils import DatasetValidationError + + +class _ASTFeatureExtractor: + sampling_rate = 16_000 + + def __call__(self, waveform, *, sampling_rate, return_tensors): + assert waveform.shape == (16_000,) + assert sampling_rate == self.sampling_rate + assert return_tensors == "pt" + return {"input_values": torch.ones((1, 1024, 128), dtype=torch.float32)} + + +class _CountingASTModel: + io_config: ClassVar = { + "input_names": ["input_values"], + "input_shapes": [[1, 1024, 128]], + } + config = SimpleNamespace( + id2label={0: "Speech", 1: "Music", 2: "Dog"}, + label2id={"Speech": 0, "Music": 1, "Dog": 2}, + ) + + def __init__(self) -> None: + self.forward_count = 0 + + def __call__(self, **inputs): + assert inputs["input_values"].shape == (1, 1024, 128) + logits = ( + torch.tensor([[8.0, 7.0, -8.0]]) + if self.forward_count == 0 + else torch.tensor([[-8.0, 7.0, 8.0]]) + ) + self.forward_count += 1 + return {"logits": logits} + + +class _IdentityWaveformExtractor: + sampling_rate = 16_000 + + def __init__(self) -> None: + self.last_waveform = None + + def __call__(self, waveform, *, sampling_rate, return_tensors, **_kwargs): + self.last_waveform = waveform + assert sampling_rate == self.sampling_rate + assert return_tensors == "pt" + return {"input_values": torch.as_tensor(waveform[None, :], dtype=torch.float32)} + + +class _BinaryModel: + config = SimpleNamespace( + id2label={0: "negative", 1: "positive"}, + label2id={"negative": 0, "positive": 1}, + ) + + def __init__(self) -> None: + self.forward_count = 0 + + def __call__(self, **inputs): + score = inputs["input_values"].mean() + self.forward_count += 1 + return {"logits": torch.stack((-score, score)).reshape(1, 2)} + + +class _EncodingMustNotRunAudio(Audio): + def encode_example(self, value): + raise AssertionError("bounded selection must not re-encode raw audio") + + +class _RawAudioIterableDataset(IterableDataset): + def __init__(self, rows, features) -> None: + self._rows = rows + self._raw_features = features + + @property + def features(self): + return self._raw_features + + @property + def column_names(self): + return list(self._raw_features) + + def take(self, count): + return iter(self._rows[:count]) + + +def test_streaming_raw_audio_is_bounded_without_feature_reencoding() -> None: + def wav_bytes(value: float) -> bytes: + buffer = BytesIO() + sf.write(buffer, np.full(16_000, value, dtype=np.float32), 16_000, format="WAV") + return buffer.getvalue() + + rows = [ + {"audio": {"bytes": wav_bytes(-0.5), "path": None}, "labels": ["negative"]}, + {"audio": {"bytes": wav_bytes(0.5), "path": None}, "labels": ["positive"]}, + ] + dataset = _RawAudioIterableDataset( + rows, + Features( + { + "audio": _EncodingMustNotRunAudio(decode=False), + "labels": Sequence(Value("string")), + } + ), + ) + config = WinMLEvaluationConfig( + model_id="example/streaming-audio", + task="audio-classification", + runtime="pytorch", + dataset=DatasetConfig( + path="example/streaming-audio", + split="test", + streaming=True, + samples=2, + shuffle=False, + columns_mapping={"label_column": "labels"}, + ), + ) + model = _BinaryModel() + extractor = _IdentityWaveformExtractor() + decoded_payloads = [] + decode_audio = _AudioModelAdapter._decode_audio + + def track_decode(audio): + decoded_payloads.append(audio) + return decode_audio(audio) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=extractor, + ), + patch.object(_AudioModelAdapter, "_decode_audio", side_effect=track_decode), + ): + metrics = WinMLAudioClassificationEvaluator(config, model).compute() + + assert [payload["bytes"] for payload in decoded_payloads] == [ + row["audio"]["bytes"] for row in rows + ] + assert model.forward_count == 2 + assert metrics["requested_samples"] == 2 + assert metrics["selected_samples"] == 2 + assert metrics["processed_samples"] == 2 + assert metrics["rejected_samples"] == 0 + assert np.isfinite(metrics["sample_average_precision"]) + assert np.isfinite(metrics["micro_average_precision"]) + + +def test_rank_three_multilabel_uses_one_forward_per_row_and_finite_ap() -> None: + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "labels": Sequence(Value("string")), + "human_labels": Sequence(Value("string")), + } + ) + dataset = Dataset.from_list( + [ + { + "audio": {"array": [0.0] * 16_000, "sampling_rate": 16_000}, + "labels": ["/m/speech", "/m/music"], + "human_labels": ["Speech", "Music"], + }, + { + "audio": {"array": [0.0] * 16_000, "sampling_rate": 16_000}, + "labels": ["/m/music", "/m/dog"], + "human_labels": ["Music", "Dog"], + }, + ], + features=features, + ) + config = WinMLEvaluationConfig( + model_id="example/ast", + task="audio-classification", + dataset=DatasetConfig( + path="example/audioset", + split="test", + samples=2, + shuffle=False, + columns_mapping={"label_column": "labels"}, + ), + ) + model = _CountingASTModel() + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_ASTFeatureExtractor(), + ), + ): + metrics = WinMLAudioClassificationEvaluator(config, model).compute() + + assert model.forward_count == 2 + assert metrics["processed_samples"] == 2 + assert np.isfinite(metrics["sample_average_precision"]) + assert np.isfinite(metrics["micro_average_precision"]) + assert 0.0 <= metrics["sample_average_precision"] <= 1.0 + assert 0.0 <= metrics["micro_average_precision"] <= 1.0 + assert metrics["requested_samples"] == 2 + assert metrics["selected_samples"] == 2 + assert metrics["rejected_samples"] == 0 + + +def test_scalar_binary_classlabel_preserves_argmax_accuracy_and_f1() -> None: + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "label": ClassLabel(names=["negative", "positive"]), + } + ) + dataset = Dataset.from_list( + [ + {"audio": {"array": [-1.0] * 8, "sampling_rate": 16_000}, "label": 0}, + {"audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, "label": 1}, + ], + features=features, + ) + config = WinMLEvaluationConfig( + model_id="example/binary-audio", + task="audio-classification", + runtime="pytorch", + dataset=DatasetConfig(path="example/binary", split="test", samples=2, shuffle=False), + ) + model = _BinaryModel() + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityWaveformExtractor(), + ), + ): + metrics = WinMLAudioClassificationEvaluator(config, model).compute() + + assert model.forward_count == 2 + assert metrics["accuracy"] == 1.0 + assert metrics["macro_f1"] == 1.0 + assert metrics["represented_classes"] == 2 + assert metrics["class_coverage"] == 1.0 + + +def test_scalar_string_labels_use_explicit_checkpoint_id_mapping() -> None: + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "gender": Value("string"), + } + ) + dataset = Dataset.from_list( + [ + {"audio": {"array": [-1.0] * 8, "sampling_rate": 16_000}, "gender": "female"}, + {"audio": {"array": [1.0] * 8, "sampling_rate": 16_000}, "gender": "male"}, + ], + features=features, + ) + config = WinMLEvaluationConfig( + model_id="example/gender-audio", + task="audio-classification", + runtime="pytorch", + dataset=DatasetConfig( + path="example/gender-audio", + split="test", + samples=2, + shuffle=False, + columns_mapping={"label_column": "gender"}, + label_mapping={"female": 0, "male": 1}, + ), + ) + model = _BinaryModel() + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityWaveformExtractor(), + ), + ): + metrics = WinMLAudioClassificationEvaluator(config, model).compute() + + assert model.forward_count == 2 + assert metrics["accuracy"] == 1.0 + assert metrics["macro_f1"] == 1.0 + + +@pytest.mark.parametrize( + ("label_mapping", "message"), + [ + (None, "non-empty explicit label mapping"), + ({}, "non-empty explicit label mapping"), + ({"female": "0", "male": 1}, "destinations must be checkpoint IDs"), + ({"female": 0, "male": 0}, "duplicate checkpoint ID destinations"), + ({"female": 2, "male": 3}, "absent from model.config.id2label"), + ({"female": 0}, "must cover every checkpoint ID"), + ], + ids=[ + "missing", + "empty", + "non-integer-destination", + "duplicate-destination", + "unknown-destinations", + "incomplete", + ], +) +def test_scalar_string_label_mapping_rejects_malformed_mappings( + label_mapping, + message, +) -> None: + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "gender": Value("string"), + } + ) + dataset = Dataset.from_list( + [{"audio": {"array": [0.0], "sampling_rate": 16_000}, "gender": "female"}], + features=features, + ) + config = WinMLEvaluationConfig( + model_id="example/gender-audio", + task="audio-classification", + dataset=DatasetConfig( + path="example/gender-audio", + samples=1, + shuffle=False, + columns_mapping={"label_column": "gender"}, + label_mapping=label_mapping, + ), + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + pytest.raises(DatasetValidationError, match=message), + ): + WinMLAudioClassificationEvaluator(config, _BinaryModel()) + + +def test_scalar_string_label_mapping_rejects_unmapped_observed_value() -> None: + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "gender": Value("string"), + } + ) + dataset = Dataset.from_list( + [ + {"audio": {"array": [-1.0], "sampling_rate": 16_000}, "gender": "female"}, + {"audio": {"array": [1.0], "sampling_rate": 16_000}, "gender": "unknown"}, + ], + features=features, + ) + config = WinMLEvaluationConfig( + model_id="example/gender-audio", + task="audio-classification", + dataset=DatasetConfig( + path="example/gender-audio", + samples=2, + shuffle=False, + columns_mapping={"label_column": "gender"}, + label_mapping={"female": 0, "male": 1}, + ), + ) + model = _BinaryModel() + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityWaveformExtractor(), + ), + pytest.raises(DatasetValidationError, match=r"'unknown'.*absent"), + ): + WinMLAudioClassificationEvaluator(config, model).compute() + + assert model.forward_count == 0 + + +def test_native_hf_model_without_io_config_uses_shared_adapter() -> None: + config = WinMLEvaluationConfig( + model_id="example/native-audio", + task="audio-classification", + runtime="pytorch", + ) + model = _BinaryModel() + extractor = _IdentityWaveformExtractor() + + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=extractor, + ): + adapter = _AudioModelAdapter(config, model) + logits = adapter.predict_logits( + {"array": np.ones(8, dtype=np.float32), "sampling_rate": 16_000} + ) + + assert model.forward_count == 1 + assert logits.shape == (2,) + + +def test_encoded_stereo_audio_is_downmixed_and_resampled(tmp_path) -> None: + audio_path = tmp_path / "stereo.wav" + frames = np.column_stack( + ( + np.full(8_000, 0.25, dtype=np.float32), + np.full(8_000, 0.75, dtype=np.float32), + ) + ) + sf.write(audio_path, frames, 8_000) + config = WinMLEvaluationConfig(model_id="example/audio", task="audio-classification") + model = _BinaryModel() + extractor = _IdentityWaveformExtractor() + + with patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=extractor, + ): + adapter = _AudioModelAdapter(config, model) + adapter.predict_logits({"bytes": None, "path": str(audio_path)}) + + assert extractor.last_waveform.shape == (16_000,) + np.testing.assert_allclose(extractor.last_waveform[100:-100], 0.5, atol=2e-3) + + +def test_saved_dataset_dict_selects_requested_split() -> None: + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "label": ClassLabel(names=["negative", "positive"]), + } + ) + train = Dataset.from_list( + [{"audio": {"array": [-1.0], "sampling_rate": 16_000}, "label": 0}], + features=features, + ) + test = Dataset.from_list( + [{"audio": {"array": [1.0], "sampling_rate": 16_000}, "label": 1}], + features=features, + ) + config = WinMLEvaluationConfig( + model_id="example/audio", + task="audio-classification", + dataset=DatasetConfig(path="saved-dataset", split="test", samples=1, shuffle=False), + ) + + with ( + patch("pathlib.Path.is_dir", return_value=True), + patch("datasets.load_from_disk", return_value=DatasetDict(train=train, test=test)), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityWaveformExtractor(), + ), + ): + evaluator = WinMLAudioClassificationEvaluator(config, _BinaryModel()) + + assert evaluator.data[0]["label"] == 1 + + +@pytest.mark.parametrize( + "label_feature", + [Value("int64"), Sequence(Sequence(Value("string")))], + ids=["ambiguous-scalar-integer", "malformed-nested-sequence"], +) +def test_ambiguous_or_malformed_target_schema_fails_closed(label_feature) -> None: + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "label": label_feature, + } + ) + label = 0 if isinstance(label_feature, Value) else [["negative"]] + dataset = Dataset.from_list( + [{"audio": {"array": [0.0], "sampling_rate": 16_000}, "label": label}], + features=features, + ) + config = WinMLEvaluationConfig( + model_id="example/audio", + task="audio-classification", + dataset=DatasetConfig(path="example/audio", samples=1, shuffle=False), + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + pytest.raises(DatasetValidationError, match="must be ClassLabel or a sequence"), + ): + WinMLAudioClassificationEvaluator(config, _BinaryModel()) + + +def test_unmapped_sequence_target_fails_as_ambiguous() -> None: + features = Features( + { + "audio": { + "array": Sequence(Value("float32")), + "sampling_rate": Value("int32"), + }, + "labels": Sequence(Value("string")), + } + ) + dataset = Dataset.from_list( + [ + { + "audio": {"array": [0.0], "sampling_rate": 16_000}, + "labels": ["/m/not-a-model-label"], + } + ], + features=features, + ) + config = WinMLEvaluationConfig( + model_id="example/audio", + task="audio-classification", + dataset=DatasetConfig( + path="example/audio", + samples=1, + shuffle=False, + columns_mapping={"label_column": "labels"}, + ), + ) + + with ( + patch("datasets.load_dataset", return_value=dataset), + patch( + "transformers.AutoFeatureExtractor.from_pretrained", + return_value=_IdentityWaveformExtractor(), + ), + pytest.raises(DatasetValidationError, match="no exact model-label match"), + ): + WinMLAudioClassificationEvaluator(config, _BinaryModel()).compute() + + +def test_registry_schema_and_no_universal_default() -> None: + from winml.modelkit.eval.evaluate import _DEFAULT_DATASETS, get_evaluator_class + from winml.modelkit.utils.eval_utils import TASK_SCHEMAS + + assert "audio-classification" in TASK_SCHEMAS + assert "audio-classification" not in _DEFAULT_DATASETS + label_schema = next( + item for item in TASK_SCHEMAS["audio-classification"].columns if item.name == "label_column" + ) + assert "explicitly mapped scalar string" in label_schema.description + assert get_evaluator_class(WinMLEvaluationConfig(task="audio-classification")) is ( + WinMLAudioClassificationEvaluator + )