From b41227f61c3f9034f6f465b1c9b82e3238f142cd Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Sun, 23 Aug 2026 04:49:27 +0800 Subject: [PATCH 1/8] Add CTC speech recognition evaluation. --- ...omatic-speech-recognition_fp16_config.json | 17 + ...omatic-speech-recognition_fp32_config.json | 17 + src/winml/modelkit/commands/build.py | 2 + src/winml/modelkit/eval/__init__.py | 3 + src/winml/modelkit/eval/ctc_asr_evaluator.py | 378 ++++++++++++++++++ src/winml/modelkit/eval/evaluate.py | 14 + src/winml/modelkit/models/winml/__init__.py | 1 + src/winml/modelkit/utils/eval_utils.py | 18 + tests/unit/commands/test_build.py | 52 +++ tests/unit/eval/test_ctc_asr_evaluator.py | 277 +++++++++++++ 10 files changed, 779 insertions(+) create mode 100644 src/winml/modelkit/eval/ctc_asr_evaluator.py create mode 100644 tests/unit/eval/test_ctc_asr_evaluator.py diff --git a/examples/recipes/facebook_mms-1b-all/cpu/cpu/automatic-speech-recognition_fp16_config.json b/examples/recipes/facebook_mms-1b-all/cpu/cpu/automatic-speech-recognition_fp16_config.json index f2d556b8f..700db8c53 100644 --- a/examples/recipes/facebook_mms-1b-all/cpu/cpu/automatic-speech-recognition_fp16_config.json +++ b/examples/recipes/facebook_mms-1b-all/cpu/cpu/automatic-speech-recognition_fp16_config.json @@ -58,5 +58,22 @@ "task": "automatic-speech-recognition", "model_class": "AutoModelForCTC", "model_type": "wav2vec2" + }, + "eval": { + "task": "automatic-speech-recognition", + "dataset": { + "path": "google/fleurs", + "name": "en_us", + "split": "validation", + "samples": 2, + "shuffle": false, + "seed": 42, + "streaming": false, + "revision": "70bb2e84b976b7e960aa89f1c648e09c59f894dd", + "columns_mapping": { + "input_column": "audio", + "label_column": "transcription" + } + } } } \ No newline at end of file diff --git a/examples/recipes/facebook_mms-1b-all/cpu/cpu/automatic-speech-recognition_fp32_config.json b/examples/recipes/facebook_mms-1b-all/cpu/cpu/automatic-speech-recognition_fp32_config.json index bc5c3f1d5..1a77e96be 100644 --- a/examples/recipes/facebook_mms-1b-all/cpu/cpu/automatic-speech-recognition_fp32_config.json +++ b/examples/recipes/facebook_mms-1b-all/cpu/cpu/automatic-speech-recognition_fp32_config.json @@ -39,5 +39,22 @@ "task": "automatic-speech-recognition", "model_class": "AutoModelForCTC", "model_type": "wav2vec2" + }, + "eval": { + "task": "automatic-speech-recognition", + "dataset": { + "path": "google/fleurs", + "name": "en_us", + "split": "validation", + "samples": 2, + "shuffle": false, + "seed": 42, + "streaming": false, + "revision": "70bb2e84b976b7e960aa89f1c648e09c59f894dd", + "columns_mapping": { + "input_column": "audio", + "label_column": "transcription" + } + } } } diff --git a/src/winml/modelkit/commands/build.py b/src/winml/modelkit/commands/build.py index a56d9f52e..8de07d6d0 100644 --- a/src/winml/modelkit/commands/build.py +++ b/src/winml/modelkit/commands/build.py @@ -2068,6 +2068,7 @@ def _build_hf_pipeline( max_iters: int = extra_kwargs.pop("hack_max_optim_iterations", 3) allow_unsupported_nodes: bool = extra_kwargs.pop("allow_unsupported_nodes", False) + skip_optimize: bool = extra_kwargs.pop("skip_optimize", config.skip_optimize) model_label = model_id or "random-init" # ── Validate + setup ───────────────────────────────────────── @@ -2153,6 +2154,7 @@ def _name(base: str) -> str: show_io_first=False, analyze_output_path=analyze_result_path, allow_unsupported_nodes=allow_unsupported_nodes, + skip_optimize=skip_optimize, ) # Persist config after autoconf diff --git a/src/winml/modelkit/eval/__init__.py b/src/winml/modelkit/eval/__init__.py index 435601a15..74eb4f949 100644 --- a/src/winml/modelkit/eval/__init__.py +++ b/src/winml/modelkit/eval/__init__.py @@ -19,6 +19,7 @@ if TYPE_CHECKING: + from .ctc_asr_evaluator import WinMLCTCASREvaluator from .depth_estimation_evaluator import WinMLDepthEstimationEvaluator from .feature_extraction_evaluator import WinMLFeatureExtractionEvaluator from .fill_mask_evaluator import WinMLFillMaskEvaluator @@ -47,6 +48,7 @@ _LAZY_ATTRS: dict[str, str] = { # Evaluators + "WinMLCTCASREvaluator": ".ctc_asr_evaluator:WinMLCTCASREvaluator", "WinMLDepthEstimationEvaluator": ".depth_estimation_evaluator:WinMLDepthEstimationEvaluator", "WinMLFeatureExtractionEvaluator": ( ".feature_extraction_evaluator:WinMLFeatureExtractionEvaluator" @@ -126,6 +128,7 @@ def __dir__() -> list[str]: "SpearmanCorrelationMetric", "TensorSimilarityEvaluator", "TopKAccuracyMetric", + "WinMLCTCASREvaluator", "WinMLDepthEstimationEvaluator", "WinMLEvaluationConfig", "WinMLEvaluator", diff --git a/src/winml/modelkit/eval/ctc_asr_evaluator.py b/src/winml/modelkit/eval/ctc_asr_evaluator.py new file mode 100644 index 000000000..90b1278fe --- /dev/null +++ b/src/winml/modelkit/eval/ctc_asr_evaluator.py @@ -0,0 +1,378 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +"""Metadata-driven CTC automatic speech recognition evaluation.""" + +from __future__ import annotations + +import math +import unicodedata +from collections import Counter +from io import BytesIO +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import numpy as np + +from ..utils.eval_utils import DatasetValidationError, validate_dataset_columns +from .base_evaluator import WinMLEvaluator + + +if TYPE_CHECKING: + from datasets import Dataset + + from .config import WinMLEvaluationConfig + +_MAX_WINDOWS_PER_UTTERANCE = 64 + + +class _RejectedSampleError(ValueError): + """A dataset row that cannot be scored under the ASR contract.""" + + +def _normalize_transcript(value: Any) -> str: + """Normalize Unicode and collapse whitespace without changing text semantics.""" + if not isinstance(value, str): + raise _RejectedSampleError("transcription is not a string") + return " ".join(unicodedata.normalize("NFKC", value).split()) + + +def _edit_distance(reference: list[str], prediction: list[str]) -> int: + """Return Levenshtein distance with memory bounded by the shorter sequence.""" + if len(reference) < len(prediction): + reference, prediction = prediction, reference + previous = list(range(len(prediction) + 1)) + for row, reference_item in enumerate(reference, start=1): + current = [row] + for column, prediction_item in enumerate(prediction, start=1): + current.append( + min( + current[-1] + 1, + previous[column] + 1, + previous[column - 1] + (reference_item != prediction_item), + ) + ) + previous = current + return previous[-1] + + +def _corpus_error_rate(references: list[str], predictions: list[str], *, words: bool) -> float: + """Compute corpus word or character error rate.""" + if len(references) != len(predictions) or not references: + raise ValueError("WER/CER require equally sized, non-empty reference and prediction lists.") + reference_units = [text.split() if words else list(text) for text in references] + prediction_units = [text.split() if words else list(text) for text in predictions] + denominator = sum(len(units) for units in reference_units) + if denominator == 0: + raise ValueError("WER/CER cannot score empty normalized references.") + errors = sum( + _edit_distance(reference, prediction) + for reference, prediction in zip(reference_units, prediction_units, strict=True) + ) + return errors / denominator + + +def _decode_audio(value: Any) -> tuple[np.ndarray, int]: + """Decode a datasets ``Audio(decode=False)`` bytes/path value with SoundFile.""" + if not isinstance(value, dict): + raise _RejectedSampleError("audio must be a decode=False bytes/path mapping") + source: BytesIO | str + audio_bytes = value.get("bytes") + audio_path = value.get("path") + if isinstance(audio_bytes, bytes): + source = BytesIO(audio_bytes) + elif isinstance(audio_path, str) and audio_path: + source = audio_path + else: + raise _RejectedSampleError("audio has neither bytes nor path") + + try: + import soundfile as sf + + waveform, sampling_rate = sf.read(source, dtype="float32", always_2d=False) + except Exception as error: + raise _RejectedSampleError(f"SoundFile decode failed: {error}") from error + waveform = np.asarray(waveform, dtype=np.float32) + if waveform.ndim == 2: + waveform = waveform.mean(axis=1, dtype=np.float32) + if waveform.ndim != 1 or waveform.size == 0: + raise _RejectedSampleError(f"decoded audio has invalid shape {waveform.shape}") + return waveform, int(sampling_rate) + + +def _resample_audio( + waveform: np.ndarray, + source_rate: int, + target_rate: int, +) -> np.ndarray: + """Resample mono audio only when processor metadata requests another rate.""" + if source_rate == target_rate: + return waveform + if source_rate <= 0 or target_rate <= 0: + raise _RejectedSampleError("audio sampling rates must be positive") + from scipy.signal import resample_poly + + divisor = math.gcd(source_rate, target_rate) + return np.asarray( + resample_poly(waveform, target_rate // divisor, source_rate // divisor), + dtype=np.float32, + ) + + +def _is_ctc_config(config: Any) -> bool: + architectures = getattr(config, "architectures", None) or [] + return any( + isinstance(architecture, str) + and (architecture.endswith("ForCTC") or architecture == "AutoModelForCTC") + for architecture in architectures + ) + + +def _configure_processor_language(processor: Any, model_config: Any) -> str | None: + """Select and validate a tokenizer language using checkpoint metadata only.""" + tokenizer = getattr(processor, "tokenizer", processor) + configured = getattr(model_config, "target_lang", None) or getattr( + model_config, + "adapter_lang", + None, + ) + active = getattr(tokenizer, "target_lang", None) + if configured is not None and configured != active: + setter = getattr(tokenizer, "set_target_lang", None) + if not callable(setter): + raise ValueError( + f"Checkpoint requests target language {configured!r}, but its tokenizer " + "cannot select a language." + ) + setter(configured) + active = getattr(tokenizer, "target_lang", configured) + if hasattr(tokenizer, "target_lang") and not active: + raise ValueError("The checkpoint tokenizer exposes target_lang but no language is active.") + return str(active) if active is not None else None + + +class WinMLCTCASREvaluator(WinMLEvaluator): + """Evaluate metadata-resolved CTC ASR models with bounded full utterances.""" + + def __init__(self, config: WinMLEvaluationConfig, model: Any) -> None: + mapping = config.dataset.columns_mapping + self._audio_column = mapping.get("input_column", "audio") + self._transcription_column = mapping.get("label_column", "transcription") + model_config = getattr(model, "config", None) + if model_config is None or not _is_ctc_config(model_config): + raise ValueError( + "automatic-speech-recognition evaluation currently supports only " + "metadata-resolved *ForCTC checkpoints." + ) + if not config.model_id: + raise ValueError("CTC ASR evaluation requires model_id to load its processor.") + + from transformers import AutoProcessor + + self.processor = AutoProcessor.from_pretrained( + config.model_id, + trust_remote_code=config.trust_remote_code, + ) + self.target_lang = _configure_processor_language(self.processor, model_config) + feature_extractor = getattr(self.processor, "feature_extractor", None) + sampling_rate = getattr(feature_extractor, "sampling_rate", None) + if not isinstance(sampling_rate, int) or sampling_rate <= 0: + raise ValueError( + "The checkpoint processor has no valid feature-extractor sampling rate." + ) + self.sampling_rate = sampling_rate + self.blank_token_id = self._resolve_blank_token_id(model_config) + self.tokenizer_vocab_size = self._resolve_tokenizer_vocab_size() + self.model = model + self.config = config + self.data = self.prepare_data() + + def prepare_data(self) -> Dataset: + """Load deterministic rows and force explicit decode=False audio values.""" + from datasets import Audio, Dataset, load_dataset, load_from_disk + + ds = self.config.dataset + try: + ds_path = Path(ds.path).expanduser() if ds.path else None + if ds_path and ds_path.is_dir(): + dataset = load_from_disk(str(ds_path)) + else: + dataset = load_dataset( + ds.path, + name=ds.name, + split=ds.split, + streaming=ds.streaming, + revision=ds.revision, + ) + dataset = dataset.cast_column(self._audio_column, Audio(decode=False)) + except Exception as error: + raise DatasetValidationError( + f"Failed to load ASR dataset '{ds.path}' " + f"(name={ds.name!r}, split='{ds.split}'): {error}" + ) from error + + validate_dataset_columns(dataset, "automatic-speech-recognition", ds.columns_mapping) + if ds.streaming: + rows = list(dataset.take(ds.samples)) + if rows and "id" in rows[0]: + rows.sort(key=lambda row: row["id"]) + return Dataset.from_list(rows) + if "id" in dataset.column_names: + dataset = dataset.sort("id") + return dataset.select(range(min(ds.samples, len(dataset)))) + + def compute(self) -> dict[str, Any]: + """Decode bounded rows and report corpus WER/CER plus accounting.""" + predictions: list[str] = [] + references: list[str] = [] + rejection_reasons: Counter[str] = Counter() + + for row in self.data: + try: + reference = _normalize_transcript(row.get(self._transcription_column)) + if not reference: + raise _RejectedSampleError("normalized transcription is empty") + prediction = self._transcribe(row.get(self._audio_column)) + if not prediction: + raise _RejectedSampleError("decoded prediction is empty") + except _RejectedSampleError as error: + rejection_reasons[str(error)] += 1 + continue + references.append(reference) + predictions.append(prediction) + + rejected = sum(rejection_reasons.values()) + if not predictions: + raise DatasetValidationError( + "No usable ASR samples remain after validation; " + f"processed=0, rejected={rejected}, reasons={dict(rejection_reasons)}" + ) + return { + "wer": _corpus_error_rate(references, predictions, words=True), + "cer": _corpus_error_rate(references, predictions, words=False), + "predictions": predictions, + "references": references, + "processed_samples": len(predictions), + "rejected_samples": rejected, + "rejection_reasons": dict(rejection_reasons), + "target_lang": self.target_lang, + } + + def _transcribe(self, audio_value: Any) -> str: + waveform, source_rate = _decode_audio(audio_value) + waveform = _resample_audio(waveform, source_rate, self.sampling_rate) + encoded = self.processor( + waveform, + sampling_rate=self.sampling_rate, + return_tensors="np", + ) + arrays = {name: np.asarray(value) for name, value in encoded.items()} + if "input_values" not in arrays or arrays["input_values"].ndim != 2: + raise _RejectedSampleError("processor did not produce rank-2 input_values") + + input_names = list((getattr(self.model, "io_config", None) or {}).get("input_names", [])) + if not input_names: + input_names = ["input_values"] + window_size = self._fixed_waveform_size(input_names) + sample_count = arrays["input_values"].shape[1] + window_count = 1 if window_size is None else math.ceil(sample_count / window_size) + if window_count > _MAX_WINDOWS_PER_UTTERANCE: + raise _RejectedSampleError( + f"utterance requires {window_count} windows; cap is {_MAX_WINDOWS_PER_UTTERANCE}" + ) + + predicted_ids: list[int] = [] + for window_index in range(window_count): + inputs = self._window_inputs( + arrays, + input_names, + window_index=window_index, + window_size=window_size, + ) + outputs = self.model(**inputs) + logits = self._extract_logits(outputs) + if logits.ndim != 3 or logits.shape[0] != 1: + raise ValueError( + f"CTC logits must have shape [1, frames, vocab], got {logits.shape}." + ) + if logits.shape[-1] != self.tokenizer_vocab_size: + raise ValueError( + f"CTC output vocabulary {logits.shape[-1]} does not match active tokenizer " + f"vocabulary {self.tokenizer_vocab_size}." + ) + if predicted_ids: + predicted_ids.append(self.blank_token_id) + predicted_ids.extend(np.argmax(logits, axis=-1)[0].astype(int).tolist()) + + decoded = self.processor.batch_decode([predicted_ids]) + if not isinstance(decoded, list) or len(decoded) != 1: + raise ValueError("CTC processor.batch_decode must return one transcript per utterance.") + return _normalize_transcript(decoded[0]) + + def _fixed_waveform_size(self, input_names: list[str]) -> int | None: + io_config = getattr(self.model, "io_config", None) or {} + shapes = io_config.get("input_shapes", []) + try: + input_index = input_names.index("input_values") + shape = shapes[input_index] + except (ValueError, IndexError): + return None + return shape[1] if len(shape) == 2 and isinstance(shape[1], int) else None + + @staticmethod + def _window_inputs( + arrays: dict[str, np.ndarray], + input_names: list[str], + *, + window_index: int, + window_size: int | None, + ) -> dict[str, np.ndarray]: + inputs: dict[str, np.ndarray] = {} + for name in input_names: + if name not in arrays: + raise ValueError(f"Processor did not produce required ONNX input {name!r}.") + value = arrays[name] + if window_size is None: + inputs[name] = value + continue + start = window_index * window_size + chunk = value[:, start : start + window_size] + if chunk.shape[1] < window_size: + chunk = np.pad(chunk, ((0, 0), (0, window_size - chunk.shape[1]))) + inputs[name] = chunk + return inputs + + @staticmethod + def _extract_logits(outputs: Any) -> np.ndarray: + logits = ( + outputs.get("logits") + if isinstance(outputs, dict) + else getattr(outputs, "logits", None) + ) + if logits is None: + raise ValueError("CTC model output does not contain logits.") + if hasattr(logits, "detach"): + logits = logits.detach().cpu().numpy() + return np.asarray(logits) + + def _resolve_blank_token_id(self, model_config: Any) -> int: + tokenizer = getattr(self.processor, "tokenizer", self.processor) + blank_token_id = getattr(tokenizer, "pad_token_id", None) + if blank_token_id is None: + blank_token_id = getattr(model_config, "pad_token_id", None) + if not isinstance(blank_token_id, int): + raise TypeError("The active CTC tokenizer has no integer blank/pad token ID.") + return blank_token_id + + def _resolve_tokenizer_vocab_size(self) -> int: + tokenizer = getattr(self.processor, "tokenizer", self.processor) + get_vocab = getattr(tokenizer, "get_vocab", None) + vocabulary = get_vocab() if callable(get_vocab) else getattr(tokenizer, "vocab", None) + if not isinstance(vocabulary, dict) or not vocabulary: + raise TypeError("The active CTC tokenizer has no vocabulary metadata.") + return len(vocabulary) + + +__all__ = ["WinMLCTCASREvaluator"] diff --git a/src/winml/modelkit/eval/evaluate.py b/src/winml/modelkit/eval/evaluate.py index e02ac79a0..2ca403cb5 100644 --- a/src/winml/modelkit/eval/evaluate.py +++ b/src/winml/modelkit/eval/evaluate.py @@ -100,6 +100,8 @@ def _select_model_loader(config: WinMLEvaluationConfig) -> _ModelLoaderKind: "winml.modelkit.eval.mask_generation_evaluator:WinMLMaskGenerationEvaluator", "text-generation": "winml.modelkit.eval.text_generation_evaluator:WinMLTextGenerationEvaluator", + "automatic-speech-recognition": + "winml.modelkit.eval.ctc_asr_evaluator:WinMLCTCASREvaluator", } # fmt: on @@ -283,6 +285,18 @@ def _validate_pytorch_runtime_config(config: WinMLEvaluationConfig) -> None: "split": "test", "columns_mapping": {"input_column": "text"}, }, + "automatic-speech-recognition": { + "path": "google/fleurs", + "name": "en_us", + "split": "validation", + "samples": 2, + "shuffle": False, + "revision": "70bb2e84b976b7e960aa89f1c648e09c59f894dd", + "columns_mapping": { + "input_column": "audio", + "label_column": "transcription", + }, + }, } diff --git a/src/winml/modelkit/models/winml/__init__.py b/src/winml/modelkit/models/winml/__init__.py index 251c40dad..dc54eb2f3 100644 --- a/src/winml/modelkit/models/winml/__init__.py +++ b/src/winml/modelkit/models/winml/__init__.py @@ -50,6 +50,7 @@ "feature-extraction": "WinMLModelForFeatureExtraction", "sentence-similarity": "WinMLModelForFeatureExtraction", "image-feature-extraction": "WinMLModelForFeatureExtraction", + "automatic-speech-recognition": "WinMLModelForGenericTask", } # Level 2: (model_type, task) -> Specialized class (exceptions only) diff --git a/src/winml/modelkit/utils/eval_utils.py b/src/winml/modelkit/utils/eval_utils.py index f85c672b4..d340ccc7e 100644 --- a/src/winml/modelkit/utils/eval_utils.py +++ b/src/winml/modelkit/utils/eval_utils.py @@ -448,6 +448,23 @@ class TaskSchema: ), ) +_AUTOMATIC_SPEECH_RECOGNITION_SCHEMA = TaskSchema( + columns=( + SchemaItem( + "input_column", + "audio bytes or path decoded by SoundFile", + default="audio", + remap_hint="", + ), + SchemaItem( + "label_column", + "reference transcription", + default="transcription", + remap_hint="", + ), + ), +) + TASK_SCHEMAS: dict[str, TaskSchema] = { "image-classification": _IMAGE_CLASSIFICATION_SCHEMA, "text-classification": _TEXT_CLASSIFICATION_SCHEMA, @@ -468,6 +485,7 @@ class TaskSchema: "keypoint-detection": _KEYPOINT_DETECTION_SCHEMA, "mask-generation": _MASK_GENERATION_SCHEMA, "text-generation": _TEXT_GENERATION_SCHEMA, + "automatic-speech-recognition": _AUTOMATIC_SPEECH_RECOGNITION_SCHEMA, } diff --git a/tests/unit/commands/test_build.py b/tests/unit/commands/test_build.py index 37f6bde79..d894c489c 100644 --- a/tests/unit/commands/test_build.py +++ b/tests/unit/commands/test_build.py @@ -2020,6 +2020,58 @@ def test_quant_model_type_carried_into_quantize_stage( assert config.quant.model_type == "qwen3_transformer_only" mock_quantize.assert_called_once() + @patch("winml.modelkit.commands.build._run_compile_stage") + @patch("winml.modelkit.commands.build._run_quantize_stage") + @patch("winml.modelkit.commands.build._run_optimize_stage") + @patch("winml.modelkit.commands.build._show_io") + @patch("winml.modelkit.utils.console.StageLive") + @patch("winml.modelkit.export.export_onnx") + @patch("winml.modelkit.build.hf._load_model") + def test_skip_optimize_reaches_hf_optimize_stage( + self, + mock_load_model: MagicMock, + mock_export_onnx: MagicMock, + mock_stage_live: MagicMock, + mock_show_io: MagicMock, + mock_optimize: MagicMock, + mock_quantize: MagicMock, + mock_compile: MagicMock, + tmp_path: Path, + ) -> None: + from winml.modelkit.commands.build import _build_hf_pipeline + + mock_stage_live.return_value.__enter__ = MagicMock(return_value=MagicMock()) + mock_stage_live.return_value.__exit__ = MagicMock(return_value=False) + mock_load_model.return_value = MagicMock() + exported = tmp_path / "export.onnx" + mock_optimize.return_value = (exported, None) + mock_quantize.return_value = exported + mock_compile.side_effect = RuntimeError("stop-after-stages") + + config = MagicMock() + config.loader.model_type = "wav2vec2" + config.loader.task = "automatic-speech-recognition" + config.loader.model_class = "AutoModelForCTC" + config.export = MagicMock() + config.quant = None + config.skip_optimize = False + config.to_dict.return_value = {} + + with pytest.raises(RuntimeError, match="stop-after-stages"): + _build_hf_pipeline( + config=config, + model_id="org/ctc-model", + output_dir=tmp_path / "out", + rebuild=True, + cache_key=None, + ep=None, + device="cpu", + extra_kwargs={"skip_optimize": True}, + preloaded_hf_config=None, + ) + + assert mock_optimize.call_args.kwargs["skip_optimize"] is True + class TestBuildEpResolution: """--ep forwarding into config generation + the compile EP-availability gate.""" diff --git a/tests/unit/eval/test_ctc_asr_evaluator.py b/tests/unit/eval/test_ctc_asr_evaluator.py new file mode 100644 index 000000000..31df54748 --- /dev/null +++ b/tests/unit/eval/test_ctc_asr_evaluator.py @@ -0,0 +1,277 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------- + +from __future__ import annotations + +from io import BytesIO +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +import numpy as np +import pytest +import soundfile as sf + +from winml.modelkit.eval.ctc_asr_evaluator import ( + WinMLCTCASREvaluator, + _configure_processor_language, + _corpus_error_rate, + _decode_audio, + _is_ctc_config, + _normalize_transcript, + _RejectedSampleError, + _resample_audio, +) +from winml.modelkit.utils.eval_utils import TASK_SCHEMAS, DatasetValidationError + + +class _Tokenizer: + pad_token_id = 0 + + def __init__(self, target_lang: str | None = None, vocab_size: int = 3) -> None: + self.target_lang = target_lang + self._vocab = {str(index): index for index in range(vocab_size)} + + def get_vocab(self) -> dict[str, int]: + return self._vocab + + def set_target_lang(self, target_lang: str) -> None: + self.target_lang = target_lang + + +class _Processor: + def __init__(self, *, target_lang: str | None = None, vocab_size: int = 3) -> None: + self.tokenizer = _Tokenizer(target_lang, vocab_size) + self.feature_extractor = SimpleNamespace(sampling_rate=16_000) + self.decoded_ids: list[int] = [] + + def __call__(self, waveform, **_kwargs): + return {"input_values": np.asarray(waveform, dtype=np.float32)[None, :]} + + def batch_decode(self, sequences): + self.decoded_ids = list(sequences[0]) + collapsed: list[int] = [] + previous = None + for token_id in self.decoded_ids: + if token_id != 0 and token_id != previous: + collapsed.append(token_id) + previous = token_id + return [" ".join(str(token_id) for token_id in collapsed)] + + +def _wav_bytes(samples: np.ndarray, sampling_rate: int = 16_000) -> bytes: + buffer = BytesIO() + sf.write(buffer, samples, sampling_rate, format="WAV", subtype="FLOAT") + return buffer.getvalue() + + +def _evaluator(*, input_shape: list[object], processor: _Processor | None = None): + evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) + evaluator.processor = processor or _Processor() + evaluator.sampling_rate = 16_000 + evaluator.blank_token_id = 0 + evaluator.tokenizer_vocab_size = 3 + evaluator.target_lang = evaluator.processor.tokenizer.target_lang + evaluator._audio_column = "audio" + evaluator._transcription_column = "transcription" + evaluator.model = MagicMock() + evaluator.model.io_config = { + "input_names": ["input_values"], + "input_shapes": [input_shape], + } + return evaluator + + +def test_asr_registry_schema_and_generic_model_route() -> None: + from winml.modelkit.eval import WinMLEvaluationConfig, get_evaluator_class + from winml.modelkit.inference import TASK_REGISTRY + from winml.modelkit.models.winml import get_winml_class + from winml.modelkit.models.winml.base import WinMLModelForGenericTask + + config = WinMLEvaluationConfig(task="automatic-speech-recognition") + assert get_evaluator_class(config) is WinMLCTCASREvaluator + assert "automatic-speech-recognition" in TASK_SCHEMAS + assert ( + get_winml_class("wav2vec2", "automatic-speech-recognition") + is WinMLModelForGenericTask + ) + assert TASK_REGISTRY["automatic-speech-recognition"].user_inputs[0].type == "audio" + + +def test_asr_default_dataset_is_pinned_and_bounded() -> None: + from winml.modelkit.eval.evaluate import _DEFAULT_DATASETS + + default = _DEFAULT_DATASETS["automatic-speech-recognition"] + assert default["path"] == "google/fleurs" + assert default["name"] == "en_us" + assert default["revision"] == "70bb2e84b976b7e960aa89f1c648e09c59f894dd" + assert default["samples"] == 2 + assert default["shuffle"] is False + + +@pytest.mark.parametrize("architecture", ["Wav2Vec2ForCTC", "HubertForCTC", "AutoModelForCTC"]) +def test_ctc_architectures_are_metadata_driven(architecture: str) -> None: + assert _is_ctc_config(SimpleNamespace(architectures=[architecture])) + + +def test_non_ctc_asr_fails_closed() -> None: + assert not _is_ctc_config(SimpleNamespace(architectures=["WhisperForConditionalGeneration"])) + + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + + model = MagicMock() + model.config = SimpleNamespace(architectures=["WhisperForConditionalGeneration"]) + config = WinMLEvaluationConfig( + model_id="org/whisper-model", + task="automatic-speech-recognition", + dataset=DatasetConfig(path="dataset"), + ) + with pytest.raises(ValueError, match=r"only metadata-resolved \*ForCTC"): + WinMLCTCASREvaluator(config, model) + + +def test_prepare_data_sorts_by_id_and_caps_rows() -> None: + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + + dataset = MagicMock() + dataset.column_names = ["id", "audio", "transcription"] + dataset.cast_column.return_value = dataset + dataset.sort.return_value = dataset + dataset.select.return_value = "selected" + dataset.__len__.return_value = 5 + evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) + evaluator._audio_column = "audio" + evaluator.config = WinMLEvaluationConfig( + task="automatic-speech-recognition", + dataset=DatasetConfig(path="dataset", samples=2, shuffle=False), + ) + + with patch("datasets.load_dataset", return_value=dataset): + assert evaluator.prepare_data() == "selected" + + dataset.sort.assert_called_once_with("id") + selected_range = dataset.select.call_args.args[0] + assert list(selected_range) == [0, 1] + + +def test_soundfile_decode_mixes_stereo_to_mono_float32() -> None: + stereo = np.column_stack( + [np.linspace(-1, 1, 32, dtype=np.float32), np.linspace(1, -1, 32, dtype=np.float32)] + ) + waveform, sampling_rate = _decode_audio({"bytes": _wav_bytes(stereo), "path": None}) + assert sampling_rate == 16_000 + assert waveform.shape == (32,) + assert waveform.dtype == np.float32 + np.testing.assert_allclose(waveform, 0.0, atol=1e-6) + + +def test_soundfile_decode_rejects_invalid_schema() -> None: + with pytest.raises(_RejectedSampleError, match="neither bytes nor path"): + _decode_audio({"bytes": None, "path": None}) + + +def test_resample_uses_processor_rate_only_when_needed() -> None: + waveform = np.arange(80, dtype=np.float32) + assert _resample_audio(waveform, 16_000, 16_000) is waveform + assert _resample_audio(waveform, 8_000, 16_000).shape == (160,) + + +def test_language_selection_uses_config_and_tokenizer_semantics() -> None: + processor = _Processor(target_lang="eng") + active = _configure_processor_language(processor, SimpleNamespace(target_lang="deu")) + assert active == "deu" + assert processor.tokenizer.target_lang == "deu" + + +def test_processor_published_language_is_preserved_without_config_override() -> None: + processor = _Processor(target_lang="eng") + assert _configure_processor_language(processor, SimpleNamespace()) == "eng" + + +def test_static_audio_windows_pad_and_insert_blank_for_processor_ctc_decode() -> None: + processor = _Processor() + evaluator = _evaluator(input_shape=[1, 4], processor=processor) + logits = np.array([[[0, 5, 0], [0, 5, 0]]], dtype=np.float32) + evaluator.model.side_effect = [ + {"logits": logits}, + {"logits": logits}, + ] + audio = {"bytes": _wav_bytes(np.arange(6, dtype=np.float32)), "path": None} + + assert evaluator._transcribe(audio) == "1 1" + assert len(evaluator.model.call_args_list) == 2 + final_input = evaluator.model.call_args_list[1].kwargs["input_values"] + assert final_input.shape == (1, 4) + np.testing.assert_array_equal(final_input[0, 2:], np.zeros(2)) + assert processor.decoded_ids == [1, 1, 0, 1, 1] + + +def test_dynamic_audio_runs_full_utterance_once() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.model.return_value = { + "logits": np.array([[[0, 5, 0], [5, 0, 0], [0, 0, 5]]], dtype=np.float32) + } + audio = {"bytes": _wav_bytes(np.arange(9, dtype=np.float32)), "path": None} + + assert evaluator._transcribe(audio) == "1 2" + assert evaluator.model.call_count == 1 + assert evaluator.model.call_args.kwargs["input_values"].shape == (1, 9) + + +def test_static_audio_rejects_more_than_64_windows_before_inference() -> None: + evaluator = _evaluator(input_shape=[1, 1]) + audio = {"bytes": _wav_bytes(np.arange(65, dtype=np.float32)), "path": None} + with pytest.raises(_RejectedSampleError, match="cap is 64"): + evaluator._transcribe(audio) + evaluator.model.assert_not_called() + + +def test_vocab_mismatch_fails_closed() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.model.return_value = {"logits": np.zeros((1, 2, 4), dtype=np.float32)} + audio = {"bytes": _wav_bytes(np.ones(4, dtype=np.float32)), "path": None} + with pytest.raises(ValueError, match="does not match active tokenizer"): + evaluator._transcribe(audio) + + +def test_transcript_normalization_and_exact_wer_cer() -> None: + assert _normalize_transcript(" hello\t world ") == "hello world" + assert _corpus_error_rate(["hello world"], ["hello there"], words=True) == 0.5 + assert _corpus_error_rate(["abc"], ["adc"], words=False) == pytest.approx(1 / 3) + + +def test_compute_preserves_accounting_and_rejection_reasons() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + {"audio": "good", "transcription": " hello world "}, + {"audio": "bad", "transcription": "ignored"}, + ] + + def transcribe(value: str) -> str: + if value == "bad": + raise _RejectedSampleError("bad audio") + return "hello world" + + evaluator._transcribe = transcribe + result = evaluator.compute() + assert result["wer"] == 0.0 + assert result["cer"] == 0.0 + assert result["processed_samples"] == 1 + assert result["rejected_samples"] == 1 + assert result["rejection_reasons"] == {"bad audio": 1} + + +def test_compute_fails_closed_when_all_rows_are_rejected() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [{"audio": None, "transcription": ""}] + with pytest.raises(DatasetValidationError, match="processed=0, rejected=1"): + evaluator.compute() + + +def test_unexpected_inference_error_propagates() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [{"audio": "value", "transcription": "reference"}] + evaluator._transcribe = MagicMock(side_effect=RuntimeError("session failed")) + with pytest.raises(RuntimeError, match="session failed"): + evaluator.compute() From b8adaf1b031a665ded5834df2bbd501c46beeea4 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Sun, 23 Aug 2026 06:12:43 +0800 Subject: [PATCH 2/8] Format CTC ASR evaluator and tests --- src/winml/modelkit/eval/ctc_asr_evaluator.py | 4 +--- tests/unit/eval/test_ctc_asr_evaluator.py | 5 +---- 2 files changed, 2 insertions(+), 7 deletions(-) diff --git a/src/winml/modelkit/eval/ctc_asr_evaluator.py b/src/winml/modelkit/eval/ctc_asr_evaluator.py index 90b1278fe..a2ac22372 100644 --- a/src/winml/modelkit/eval/ctc_asr_evaluator.py +++ b/src/winml/modelkit/eval/ctc_asr_evaluator.py @@ -347,9 +347,7 @@ def _window_inputs( @staticmethod def _extract_logits(outputs: Any) -> np.ndarray: logits = ( - outputs.get("logits") - if isinstance(outputs, dict) - else getattr(outputs, "logits", None) + outputs.get("logits") if isinstance(outputs, dict) else getattr(outputs, "logits", None) ) if logits is None: raise ValueError("CTC model output does not contain logits.") diff --git a/tests/unit/eval/test_ctc_asr_evaluator.py b/tests/unit/eval/test_ctc_asr_evaluator.py index 31df54748..cef21d605 100644 --- a/tests/unit/eval/test_ctc_asr_evaluator.py +++ b/tests/unit/eval/test_ctc_asr_evaluator.py @@ -92,10 +92,7 @@ def test_asr_registry_schema_and_generic_model_route() -> None: config = WinMLEvaluationConfig(task="automatic-speech-recognition") assert get_evaluator_class(config) is WinMLCTCASREvaluator assert "automatic-speech-recognition" in TASK_SCHEMAS - assert ( - get_winml_class("wav2vec2", "automatic-speech-recognition") - is WinMLModelForGenericTask - ) + assert get_winml_class("wav2vec2", "automatic-speech-recognition") is WinMLModelForGenericTask assert TASK_REGISTRY["automatic-speech-recognition"].user_inputs[0].type == "audio" From 2952bf38896016101237aa2b6cde554709de05d8 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Sun, 23 Aug 2026 08:29:11 +0800 Subject: [PATCH 3/8] Initialize CTC evaluator base state --- src/winml/modelkit/eval/ctc_asr_evaluator.py | 9 ++++--- tests/unit/eval/test_ctc_asr_evaluator.py | 25 ++++++++++++++++++++ 2 files changed, 31 insertions(+), 3 deletions(-) diff --git a/src/winml/modelkit/eval/ctc_asr_evaluator.py b/src/winml/modelkit/eval/ctc_asr_evaluator.py index a2ac22372..e2b3a51bc 100644 --- a/src/winml/modelkit/eval/ctc_asr_evaluator.py +++ b/src/winml/modelkit/eval/ctc_asr_evaluator.py @@ -22,6 +22,7 @@ if TYPE_CHECKING: from datasets import Dataset + from transformers.pipelines.base import Pipeline from .config import WinMLEvaluationConfig @@ -185,9 +186,11 @@ def __init__(self, config: WinMLEvaluationConfig, model: Any) -> None: self.sampling_rate = sampling_rate self.blank_token_id = self._resolve_blank_token_id(model_config) self.tokenizer_vocab_size = self._resolve_tokenizer_vocab_size() - self.model = model - self.config = config - self.data = self.prepare_data() + super().__init__(config, model) + + def prepare_pipeline(self) -> Pipeline | None: # type: ignore[override] + """Skip the HF pipeline because CTC decoding drives the model directly.""" + return None def prepare_data(self) -> Dataset: """Load deterministic rows and force explicit decode=False audio values.""" diff --git a/tests/unit/eval/test_ctc_asr_evaluator.py b/tests/unit/eval/test_ctc_asr_evaluator.py index cef21d605..bf4d54f86 100644 --- a/tests/unit/eval/test_ctc_asr_evaluator.py +++ b/tests/unit/eval/test_ctc_asr_evaluator.py @@ -128,6 +128,31 @@ def test_non_ctc_asr_fails_closed() -> None: WinMLCTCASREvaluator(config, model) +def test_constructor_initializes_base_evaluator_state() -> None: + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + + processor = _Processor(target_lang="eng") + model = MagicMock() + model.config = SimpleNamespace(architectures=["Wav2Vec2ForCTC"], pad_token_id=0) + config = WinMLEvaluationConfig( + model_id="org/ctc-model", + task="automatic-speech-recognition", + dataset=DatasetConfig(path="dataset"), + ) + prepared_data = object() + + with ( + patch("transformers.AutoProcessor.from_pretrained", return_value=processor), + patch.object(WinMLCTCASREvaluator, "prepare_data", return_value=prepared_data), + ): + evaluator = WinMLCTCASREvaluator(config, model) + + assert evaluator.model is model + assert evaluator.config is config + assert evaluator.data is prepared_data + assert evaluator.pipe is None + + def test_prepare_data_sorts_by_id_and_caps_rows() -> None: from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig From e28ff22408b3f990c6454988d967ec4a53810742 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Sun, 23 Aug 2026 10:13:34 +0800 Subject: [PATCH 4/8] Count empty CTC hypotheses as deletions --- src/winml/modelkit/eval/ctc_asr_evaluator.py | 3 +- tests/unit/eval/test_ctc_asr_evaluator.py | 83 ++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) diff --git a/src/winml/modelkit/eval/ctc_asr_evaluator.py b/src/winml/modelkit/eval/ctc_asr_evaluator.py index e2b3a51bc..63ce5fc0f 100644 --- a/src/winml/modelkit/eval/ctc_asr_evaluator.py +++ b/src/winml/modelkit/eval/ctc_asr_evaluator.py @@ -238,8 +238,6 @@ def compute(self) -> dict[str, Any]: if not reference: raise _RejectedSampleError("normalized transcription is empty") prediction = self._transcribe(row.get(self._audio_column)) - if not prediction: - raise _RejectedSampleError("decoded prediction is empty") except _RejectedSampleError as error: rejection_reasons[str(error)] += 1 continue @@ -258,6 +256,7 @@ def compute(self) -> dict[str, Any]: "predictions": predictions, "references": references, "processed_samples": len(predictions), + "skipped_samples": 0, "rejected_samples": rejected, "rejection_reasons": dict(rejection_reasons), "target_lang": self.target_lang, diff --git a/tests/unit/eval/test_ctc_asr_evaluator.py b/tests/unit/eval/test_ctc_asr_evaluator.py index bf4d54f86..4ef843296 100644 --- a/tests/unit/eval/test_ctc_asr_evaluator.py +++ b/tests/unit/eval/test_ctc_asr_evaluator.py @@ -280,10 +280,93 @@ def transcribe(value: str) -> str: assert result["wer"] == 0.0 assert result["cer"] == 0.0 assert result["processed_samples"] == 1 + assert result["skipped_samples"] == 0 assert result["rejected_samples"] == 1 assert result["rejection_reasons"] == {"bad audio": 1} +def test_compute_scores_successful_empty_decode_as_deletions() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + {"audio": "empty", "transcription": "hello world"}, + {"audio": "decoded", "transcription": "good day"}, + ] + evaluator._transcribe = lambda value: "" if value == "empty" else "good day" + + result = evaluator.compute() + + assert result["wer"] == 0.5 + assert result["cer"] == pytest.approx(11 / 19) + assert result["predictions"] == ["", "good day"] + assert result["processed_samples"] == 2 + assert result["skipped_samples"] == 0 + assert result["rejected_samples"] == 0 + assert result["rejection_reasons"] == {} + + +def test_compute_scores_all_empty_hypotheses_as_deletions() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + {"audio": "first", "transcription": "hello world"}, + {"audio": "second", "transcription": "good day"}, + ] + evaluator._transcribe = lambda _value: "" + + result = evaluator.compute() + + assert result["wer"] == 1.0 + assert result["cer"] == 1.0 + assert result["predictions"] == ["", ""] + assert result["processed_samples"] == 2 + assert result["skipped_samples"] == 0 + assert result["rejected_samples"] == 0 + assert result["rejection_reasons"] == {} + + +def test_compute_rejects_empty_reference_but_keeps_empty_hypothesis() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + {"audio": "invalid-reference", "transcription": " \t "}, + {"audio": "empty-hypothesis", "transcription": "valid"}, + ] + evaluator._transcribe = lambda _value: "" + + result = evaluator.compute() + + assert result["wer"] == 1.0 + assert result["cer"] == 1.0 + assert result["predictions"] == [""] + assert result["references"] == ["valid"] + assert result["processed_samples"] == 1 + assert result["skipped_samples"] == 0 + assert result["rejected_samples"] == 1 + assert result["rejection_reasons"] == {"normalized transcription is empty": 1} + + +def test_compute_distinguishes_decode_failure_from_empty_decode() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + {"audio": "failure", "transcription": "rejected"}, + {"audio": "empty", "transcription": "kept"}, + ] + + def transcribe(value: str) -> str: + if value == "failure": + raise _RejectedSampleError("decode failed") + return "" + + evaluator._transcribe = transcribe + result = evaluator.compute() + + assert result["wer"] == 1.0 + assert result["cer"] == 1.0 + assert result["predictions"] == [""] + assert result["processed_samples"] == 1 + assert result["skipped_samples"] == 0 + assert result["rejected_samples"] == 1 + assert result["rejection_reasons"] == {"decode failed": 1} + + def test_compute_fails_closed_when_all_rows_are_rejected() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [{"audio": None, "transcription": ""}] From 6e6acf95d3060df10652ef24d93a746a0301069d Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Sun, 23 Aug 2026 14:24:30 +0800 Subject: [PATCH 5/8] Support ordinary Wav2Vec2 CTC processors --- src/winml/modelkit/eval/ctc_asr_evaluator.py | 60 ++++++++++++-- tests/unit/eval/test_ctc_asr_evaluator.py | 86 +++++++++++++++++++- 2 files changed, 136 insertions(+), 10 deletions(-) diff --git a/src/winml/modelkit/eval/ctc_asr_evaluator.py b/src/winml/modelkit/eval/ctc_asr_evaluator.py index 63ce5fc0f..527a42481 100644 --- a/src/winml/modelkit/eval/ctc_asr_evaluator.py +++ b/src/winml/modelkit/eval/ctc_asr_evaluator.py @@ -131,6 +131,36 @@ def _is_ctc_config(config: Any) -> bool: ) +def _load_ctc_processor(model_id: str, *, trust_remote_code: bool) -> Any: + """Load the published processor, falling back only from an unavailable optional LM.""" + from transformers import AutoProcessor + + try: + return AutoProcessor.from_pretrained(model_id, trust_remote_code=trust_remote_code) + except ImportError as error: + message = str(error) + if "Wav2Vec2ProcessorWithLM" not in message or "pyctcdecode" not in message: + raise + + from transformers import AutoFeatureExtractor, AutoTokenizer, Wav2Vec2Processor + + try: + feature_extractor = AutoFeatureExtractor.from_pretrained( + model_id, + trust_remote_code=trust_remote_code, + ) + tokenizer = AutoTokenizer.from_pretrained( + model_id, + trust_remote_code=trust_remote_code, + ) + return Wav2Vec2Processor(feature_extractor=feature_extractor, tokenizer=tokenizer) + except Exception as error: + raise ValueError( + "The checkpoint declares Wav2Vec2ProcessorWithLM, but optional pyctcdecode is " + "unavailable and its feature extractor/tokenizer cannot form a greedy CTC processor." + ) from error + + def _configure_processor_language(processor: Any, model_config: Any) -> str | None: """Select and validate a tokenizer language using checkpoint metadata only.""" tokenizer = getattr(processor, "tokenizer", processor) @@ -140,6 +170,17 @@ def _configure_processor_language(processor: Any, model_config: Any) -> str | No None, ) active = getattr(tokenizer, "target_lang", None) + adapter_attn_dim = getattr(model_config, "adapter_attn_dim", None) + adapter_capable = ( + isinstance(adapter_attn_dim, int) and adapter_attn_dim > 0 + ) or active is not None + if not adapter_capable: + if configured is not None: + raise ValueError( + f"Checkpoint requests target language {configured!r}, but its metadata does not " + "expose language-adapter support." + ) + return None if configured is not None and configured != active: setter = getattr(tokenizer, "set_target_lang", None) if not callable(setter): @@ -147,10 +188,19 @@ def _configure_processor_language(processor: Any, model_config: Any) -> str | No f"Checkpoint requests target language {configured!r}, but its tokenizer " "cannot select a language." ) - setter(configured) + try: + setter(configured) + except Exception as error: + raise ValueError( + f"Checkpoint tokenizer cannot select requested adapter language {configured!r}." + ) from error active = getattr(tokenizer, "target_lang", configured) - if hasattr(tokenizer, "target_lang") and not active: - raise ValueError("The checkpoint tokenizer exposes target_lang but no language is active.") + if active != configured: + raise ValueError( + f"Checkpoint tokenizer did not activate requested adapter language {configured!r}." + ) + if not active: + raise ValueError("The checkpoint supports language adapters but no language is active.") return str(active) if active is not None else None @@ -170,9 +220,7 @@ def __init__(self, config: WinMLEvaluationConfig, model: Any) -> None: if not config.model_id: raise ValueError("CTC ASR evaluation requires model_id to load its processor.") - from transformers import AutoProcessor - - self.processor = AutoProcessor.from_pretrained( + self.processor = _load_ctc_processor( config.model_id, trust_remote_code=config.trust_remote_code, ) diff --git a/tests/unit/eval/test_ctc_asr_evaluator.py b/tests/unit/eval/test_ctc_asr_evaluator.py index 4ef843296..565386e24 100644 --- a/tests/unit/eval/test_ctc_asr_evaluator.py +++ b/tests/unit/eval/test_ctc_asr_evaluator.py @@ -19,6 +19,7 @@ _corpus_error_rate, _decode_audio, _is_ctc_config, + _load_ctc_processor, _normalize_transcript, _RejectedSampleError, _resample_audio, @@ -199,16 +200,93 @@ def test_resample_uses_processor_rate_only_when_needed() -> None: assert _resample_audio(waveform, 8_000, 16_000).shape == (160,) -def test_language_selection_uses_config_and_tokenizer_semantics() -> None: +def test_optional_lm_dependency_falls_back_to_plain_wav2vec2_processor() -> None: + feature_extractor = object() + tokenizer = object() + processor = object() + missing_lm = ImportError( + "Wav2Vec2ProcessorWithLM requires the pyctcdecode library but it was not found" + ) + + with ( + patch("transformers.AutoProcessor.from_pretrained", side_effect=missing_lm), + patch("transformers.AutoFeatureExtractor.from_pretrained", return_value=feature_extractor), + patch("transformers.AutoTokenizer.from_pretrained", return_value=tokenizer), + patch("transformers.Wav2Vec2Processor", return_value=processor) as processor_class, + ): + assert _load_ctc_processor("org/ctc-model", trust_remote_code=False) is processor + + processor_class.assert_called_once_with( + feature_extractor=feature_extractor, + tokenizer=tokenizer, + ) + + +def test_available_lm_processor_is_preserved() -> None: + processor = object() + with patch("transformers.AutoProcessor.from_pretrained", return_value=processor): + assert _load_ctc_processor("org/lm-model", trust_remote_code=False) is processor + + +def test_unrelated_processor_import_error_fails_closed() -> None: + with ( + patch( + "transformers.AutoProcessor.from_pretrained", + side_effect=ImportError("unsupported custom processor dependency"), + ), + pytest.raises(ImportError, match="unsupported custom processor dependency"), + ): + _load_ctc_processor("org/unsupported-model", trust_remote_code=False) + + +def test_incompatible_greedy_processor_components_fail_clearly() -> None: + missing_lm = ImportError( + "Wav2Vec2ProcessorWithLM requires the pyctcdecode library but it was not found" + ) + with ( + patch("transformers.AutoProcessor.from_pretrained", side_effect=missing_lm), + patch("transformers.AutoFeatureExtractor.from_pretrained", return_value=object()), + patch("transformers.AutoTokenizer.from_pretrained", return_value=object()), + patch("transformers.Wav2Vec2Processor", side_effect=TypeError("incompatible components")), + pytest.raises(ValueError, match="cannot form a greedy CTC processor"), + ): + _load_ctc_processor("org/unsupported-model", trust_remote_code=False) + + +def test_ordinary_wav2vec2_tokenizer_accepts_null_target_language() -> None: + processor = _Processor(target_lang=None) + assert _configure_processor_language(processor, SimpleNamespace(adapter_attn_dim=None)) is None + + +def test_mms_language_selection_uses_adapter_metadata() -> None: processor = _Processor(target_lang="eng") - active = _configure_processor_language(processor, SimpleNamespace(target_lang="deu")) + active = _configure_processor_language( + processor, + SimpleNamespace(target_lang="deu", adapter_attn_dim=16), + ) assert active == "deu" assert processor.tokenizer.target_lang == "deu" -def test_processor_published_language_is_preserved_without_config_override() -> None: +def test_mms_published_language_is_preserved_without_config_override() -> None: + processor = _Processor(target_lang="eng") + assert _configure_processor_language(processor, SimpleNamespace(adapter_attn_dim=16)) == "eng" + + +def test_invalid_mms_adapter_fails_closed() -> None: processor = _Processor(target_lang="eng") - assert _configure_processor_language(processor, SimpleNamespace()) == "eng" + processor.tokenizer.set_target_lang = MagicMock(side_effect=ValueError("invalid adapter")) + with pytest.raises(ValueError, match="cannot select requested adapter language 'invalid'"): + _configure_processor_language( + processor, + SimpleNamespace(target_lang="invalid", adapter_attn_dim=16), + ) + + +def test_mms_adapter_metadata_requires_an_active_language() -> None: + processor = _Processor(target_lang=None) + with pytest.raises(ValueError, match="supports language adapters but no language is active"): + _configure_processor_language(processor, SimpleNamespace(adapter_attn_dim=16)) def test_static_audio_windows_pad_and_insert_blank_for_processor_ctc_decode() -> None: From a19b74d722c7c2635d7e7bf4fb286ed9a12cff6a Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Mon, 24 Aug 2026 00:40:34 +0800 Subject: [PATCH 6/8] Preserve CTC evaluation source order --- src/winml/modelkit/eval/ctc_asr_evaluator.py | 44 ++++++- tests/unit/eval/test_ctc_asr_evaluator.py | 124 +++++++++++++++---- 2 files changed, 137 insertions(+), 31 deletions(-) diff --git a/src/winml/modelkit/eval/ctc_asr_evaluator.py b/src/winml/modelkit/eval/ctc_asr_evaluator.py index 527a42481..bee4f1e89 100644 --- a/src/winml/modelkit/eval/ctc_asr_evaluator.py +++ b/src/winml/modelkit/eval/ctc_asr_evaluator.py @@ -204,6 +204,28 @@ def _configure_processor_language(processor: Any, model_config: Any) -> str | No return str(active) if active is not None else None +def _selected_row_provenance(row: dict[str, Any], audio_column: str) -> dict[str, Any]: + source_index = row.get("_winml_source_index") + if not isinstance(source_index, int) or isinstance(source_index, bool): + raise DatasetValidationError("Selected ASR row has no valid source index provenance.") + audio = row.get(audio_column) + audio_path: str | None = None + audio_key: str | int | None = None + if isinstance(audio, dict): + path = audio.get("path") + if isinstance(path, str) and path: + audio_path = Path(path).name + key = audio.get("key") + if isinstance(key, (str, int)) and not isinstance(key, bool): + audio_key = key + return { + "source_index": source_index, + "dataset_id": row.get("id"), + "audio_path": audio_path, + "audio_key": audio_key, + } + + class WinMLCTCASREvaluator(WinMLEvaluator): """Evaluate metadata-resolved CTC ASR models with bounded full utterances.""" @@ -267,18 +289,21 @@ def prepare_data(self) -> Dataset: validate_dataset_columns(dataset, "automatic-speech-recognition", ds.columns_mapping) if ds.streaming: rows = list(dataset.take(ds.samples)) - if rows and "id" in rows[0]: - rows.sort(key=lambda row: row["id"]) - return Dataset.from_list(rows) - if "id" in dataset.column_names: - dataset = dataset.sort("id") - return dataset.select(range(min(ds.samples, len(dataset)))) + else: + rows = [dataset[index] for index in range(min(ds.samples, len(dataset)))] + return Dataset.from_list( + [{**row, "_winml_source_index": index} for index, row in enumerate(rows)] + ) def compute(self) -> dict[str, Any]: """Decode bounded rows and report corpus WER/CER plus accounting.""" predictions: list[str] = [] references: list[str] = [] rejection_reasons: Counter[str] = Counter() + selected_rows = [_selected_row_provenance(row, self._audio_column) for row in self.data] + source_indices = [row["source_index"] for row in selected_rows] + if len(source_indices) != len(set(source_indices)): + raise DatasetValidationError("Selected ASR rows contain duplicate source indices.") for row in self.data: try: @@ -303,6 +328,13 @@ def compute(self) -> dict[str, Any]: "cer": _corpus_error_rate(references, predictions, words=False), "predictions": predictions, "references": references, + "requested_samples": self.config.dataset.samples, + "selected_samples": len(selected_rows), + "selected_source_ids": [row["dataset_id"] for row in selected_rows], + "selected_source_indices": source_indices, + "selected_audio_paths": [row["audio_path"] for row in selected_rows], + "selected_audio_keys": [row["audio_key"] for row in selected_rows], + "selected_rows": selected_rows, "processed_samples": len(predictions), "skipped_samples": 0, "rejected_samples": rejected, diff --git a/tests/unit/eval/test_ctc_asr_evaluator.py b/tests/unit/eval/test_ctc_asr_evaluator.py index 565386e24..2bb44e239 100644 --- a/tests/unit/eval/test_ctc_asr_evaluator.py +++ b/tests/unit/eval/test_ctc_asr_evaluator.py @@ -68,6 +68,8 @@ def _wav_bytes(samples: np.ndarray, sampling_rate: int = 16_000) -> bytes: def _evaluator(*, input_shape: list[object], processor: _Processor | None = None): + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) evaluator.processor = processor or _Processor() evaluator.sampling_rate = 16_000 @@ -76,6 +78,10 @@ def _evaluator(*, input_shape: list[object], processor: _Processor | None = None evaluator.target_lang = evaluator.processor.tokenizer.target_lang evaluator._audio_column = "audio" evaluator._transcription_column = "transcription" + evaluator.config = WinMLEvaluationConfig( + task="automatic-speech-recognition", + dataset=DatasetConfig(path="dataset", samples=2, shuffle=False), + ) evaluator.model = MagicMock() evaluator.model.io_config = { "input_names": ["input_values"], @@ -154,28 +160,34 @@ def test_constructor_initializes_base_evaluator_state() -> None: assert evaluator.pipe is None -def test_prepare_data_sorts_by_id_and_caps_rows() -> None: +@pytest.mark.parametrize("streaming", [False, True]) +def test_prepare_data_preserves_source_order_with_duplicate_ids(streaming: bool) -> None: from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig - dataset = MagicMock() - dataset.column_names = ["id", "audio", "transcription"] + rows = [ + {"id": 1525, "audio": {"path": "first.wav"}, "transcription": "first"}, + {"id": 1657, "audio": {"path": "second.wav"}, "transcription": "second"}, + {"id": 1525, "audio": {"path": "third.wav"}, "transcription": "third"}, + ] + dataset = MagicMock(column_names=["id", "audio", "transcription"]) dataset.cast_column.return_value = dataset - dataset.sort.return_value = dataset - dataset.select.return_value = "selected" - dataset.__len__.return_value = 5 + dataset.__len__.return_value = len(rows) + dataset.__getitem__.side_effect = rows.__getitem__ + dataset.take.return_value = rows evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) evaluator._audio_column = "audio" evaluator.config = WinMLEvaluationConfig( task="automatic-speech-recognition", - dataset=DatasetConfig(path="dataset", samples=2, shuffle=False), + dataset=DatasetConfig(path="dataset", samples=3, shuffle=False, streaming=streaming), ) with patch("datasets.load_dataset", return_value=dataset): - assert evaluator.prepare_data() == "selected" + selected = evaluator.prepare_data() - dataset.sort.assert_called_once_with("id") - selected_range = dataset.select.call_args.args[0] - assert list(selected_range) == [0, 1] + assert selected["id"] == [1525, 1657, 1525] + assert selected["_winml_source_index"] == [0, 1, 2] + assert selected["transcription"] == ["first", "second", "third"] + dataset.sort.assert_not_called() def test_soundfile_decode_mixes_stereo_to_mono_float32() -> None: @@ -344,12 +356,22 @@ def test_transcript_normalization_and_exact_wer_cer() -> None: def test_compute_preserves_accounting_and_rejection_reasons() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"audio": "good", "transcription": " hello world "}, - {"audio": "bad", "transcription": "ignored"}, + { + "_winml_source_index": 0, + "id": 1525, + "audio": {"path": r"C:\scratch\10018492969996036091.wav", "key": "first"}, + "transcription": " hello world ", + }, + { + "_winml_source_index": 1, + "id": 1657, + "audio": {"path": r"C:\scratch\10288018704489549018.wav", "key": "second"}, + "transcription": "ignored", + }, ] - def transcribe(value: str) -> str: - if value == "bad": + def transcribe(value: dict[str, str]) -> str: + if value["key"] == "second": raise _RejectedSampleError("bad audio") return "hello world" @@ -357,17 +379,69 @@ def transcribe(value: str) -> str: result = evaluator.compute() assert result["wer"] == 0.0 assert result["cer"] == 0.0 + assert result["requested_samples"] == 2 + assert result["selected_samples"] == 2 + assert result["selected_source_ids"] == [1525, 1657] + assert result["selected_source_indices"] == [0, 1] + assert result["selected_audio_paths"] == [ + "10018492969996036091.wav", + "10288018704489549018.wav", + ] + assert result["selected_audio_keys"] == ["first", "second"] + assert result["selected_rows"] == [ + { + "source_index": 0, + "dataset_id": 1525, + "audio_path": "10018492969996036091.wav", + "audio_key": "first", + }, + { + "source_index": 1, + "dataset_id": 1657, + "audio_path": "10288018704489549018.wav", + "audio_key": "second", + }, + ] assert result["processed_samples"] == 1 assert result["skipped_samples"] == 0 assert result["rejected_samples"] == 1 assert result["rejection_reasons"] == {"bad audio": 1} + assert ( + result["processed_samples"] + result["rejected_samples"] + result["skipped_samples"] + == result["selected_samples"] + ) + + +def test_compute_rejects_duplicate_selected_source_indices_before_inference() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + {"_winml_source_index": 0, "audio": "first", "transcription": "first"}, + {"_winml_source_index": 0, "audio": "second", "transcription": "second"}, + ] + evaluator._transcribe = MagicMock() + + with pytest.raises(DatasetValidationError, match="duplicate source indices"): + evaluator.compute() + + evaluator._transcribe.assert_not_called() + + +def test_compute_rejects_missing_selected_source_index_before_inference() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [{"audio": "value", "transcription": "reference"}] + evaluator._transcribe = MagicMock() + + with pytest.raises(DatasetValidationError, match="no valid source index provenance"): + evaluator.compute() + + evaluator._transcribe.assert_not_called() def test_compute_scores_successful_empty_decode_as_deletions() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"audio": "empty", "transcription": "hello world"}, - {"audio": "decoded", "transcription": "good day"}, + {"_winml_source_index": 0, "audio": "empty", "transcription": "hello world"}, + {"_winml_source_index": 1, "audio": "decoded", "transcription": "good day"}, ] evaluator._transcribe = lambda value: "" if value == "empty" else "good day" @@ -385,8 +459,8 @@ def test_compute_scores_successful_empty_decode_as_deletions() -> None: def test_compute_scores_all_empty_hypotheses_as_deletions() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"audio": "first", "transcription": "hello world"}, - {"audio": "second", "transcription": "good day"}, + {"_winml_source_index": 0, "audio": "first", "transcription": "hello world"}, + {"_winml_source_index": 1, "audio": "second", "transcription": "good day"}, ] evaluator._transcribe = lambda _value: "" @@ -404,8 +478,8 @@ def test_compute_scores_all_empty_hypotheses_as_deletions() -> None: def test_compute_rejects_empty_reference_but_keeps_empty_hypothesis() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"audio": "invalid-reference", "transcription": " \t "}, - {"audio": "empty-hypothesis", "transcription": "valid"}, + {"_winml_source_index": 0, "audio": "invalid-reference", "transcription": " \t "}, + {"_winml_source_index": 1, "audio": "empty-hypothesis", "transcription": "valid"}, ] evaluator._transcribe = lambda _value: "" @@ -424,8 +498,8 @@ def test_compute_rejects_empty_reference_but_keeps_empty_hypothesis() -> None: def test_compute_distinguishes_decode_failure_from_empty_decode() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"audio": "failure", "transcription": "rejected"}, - {"audio": "empty", "transcription": "kept"}, + {"_winml_source_index": 0, "audio": "failure", "transcription": "rejected"}, + {"_winml_source_index": 1, "audio": "empty", "transcription": "kept"}, ] def transcribe(value: str) -> str: @@ -447,14 +521,14 @@ def transcribe(value: str) -> str: def test_compute_fails_closed_when_all_rows_are_rejected() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) - evaluator.data = [{"audio": None, "transcription": ""}] + evaluator.data = [{"_winml_source_index": 0, "audio": None, "transcription": ""}] with pytest.raises(DatasetValidationError, match="processed=0, rejected=1"): evaluator.compute() def test_unexpected_inference_error_propagates() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) - evaluator.data = [{"audio": "value", "transcription": "reference"}] + evaluator.data = [{"_winml_source_index": 0, "audio": "value", "transcription": "reference"}] evaluator._transcribe = MagicMock(side_effect=RuntimeError("session failed")) with pytest.raises(RuntimeError, match="session failed"): evaluator.compute() From 34bc2a0e6181e8c9f0cc27fddf9ebfaf2d1dbfe7 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Mon, 24 Aug 2026 06:48:51 +0800 Subject: [PATCH 7/8] Validate CTC selected row identities --- src/winml/modelkit/eval/ctc_asr_evaluator.py | 41 ++++- tests/unit/eval/test_ctc_asr_evaluator.py | 177 +++++++++++++++++-- 2 files changed, 200 insertions(+), 18 deletions(-) diff --git a/src/winml/modelkit/eval/ctc_asr_evaluator.py b/src/winml/modelkit/eval/ctc_asr_evaluator.py index bee4f1e89..c23c9119d 100644 --- a/src/winml/modelkit/eval/ctc_asr_evaluator.py +++ b/src/winml/modelkit/eval/ctc_asr_evaluator.py @@ -204,23 +204,48 @@ def _configure_processor_language(processor: Any, model_config: Any) -> str | No return str(active) if active is not None else None +def _normalize_identity_scalar(value: Any, field: str) -> str | int | float: + if isinstance(value, np.generic): + value = value.item() + if isinstance(value, str): + value = unicodedata.normalize("NFKC", value).strip() + if value: + return str(value) + elif isinstance(value, int) and not isinstance(value, bool): + return int(value) + elif isinstance(value, float) and math.isfinite(value): + return float(value) + raise DatasetValidationError(f"Selected ASR row has no valid {field} provenance.") + + def _selected_row_provenance(row: dict[str, Any], audio_column: str) -> dict[str, Any]: source_index = row.get("_winml_source_index") - if not isinstance(source_index, int) or isinstance(source_index, bool): + if isinstance(source_index, np.integer): + source_index = int(source_index) + if not isinstance(source_index, int) or isinstance(source_index, bool) or source_index < 0: raise DatasetValidationError("Selected ASR row has no valid source index provenance.") + dataset_id = _normalize_identity_scalar(row.get("id"), "dataset ID") audio = row.get(audio_column) audio_path: str | None = None - audio_key: str | int | None = None + audio_key: str | int | float | None = None if isinstance(audio, dict): path = audio.get("path") if isinstance(path, str) and path: - audio_path = Path(path).name + audio_path = unicodedata.normalize( + "NFKC", path.replace("\\", "/").rsplit("/", 1)[-1] + ).strip() key = audio.get("key") - if isinstance(key, (str, int)) and not isinstance(key, bool): - audio_key = key + if key not in (None, ""): + audio_key = _normalize_identity_scalar(key, "audio key") + elif isinstance(audio, str) and audio: + audio_path = unicodedata.normalize( + "NFKC", audio.replace("\\", "/").rsplit("/", 1)[-1] + ).strip() + if not audio_path and audio_key is None: + raise DatasetValidationError("Selected ASR row has no valid audio identity provenance.") return { "source_index": source_index, - "dataset_id": row.get("id"), + "dataset_id": dataset_id, "audio_path": audio_path, "audio_key": audio_key, } @@ -304,6 +329,10 @@ def compute(self) -> dict[str, Any]: source_indices = [row["source_index"] for row in selected_rows] if len(source_indices) != len(set(source_indices)): raise DatasetValidationError("Selected ASR rows contain duplicate source indices.") + audio_paths = [row["audio_path"] for row in selected_rows if row["audio_path"] is not None] + audio_keys = [row["audio_key"] for row in selected_rows if row["audio_key"] is not None] + if len(audio_paths) != len(set(audio_paths)) or len(audio_keys) != len(set(audio_keys)): + raise DatasetValidationError("Selected ASR rows contain duplicate audio identities.") for row in self.data: try: diff --git a/tests/unit/eval/test_ctc_asr_evaluator.py b/tests/unit/eval/test_ctc_asr_evaluator.py index 2bb44e239..5056eebbb 100644 --- a/tests/unit/eval/test_ctc_asr_evaluator.py +++ b/tests/unit/eval/test_ctc_asr_evaluator.py @@ -415,8 +415,8 @@ def transcribe(value: dict[str, str]) -> str: def test_compute_rejects_duplicate_selected_source_indices_before_inference() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"_winml_source_index": 0, "audio": "first", "transcription": "first"}, - {"_winml_source_index": 0, "audio": "second", "transcription": "second"}, + {"_winml_source_index": 0, "id": 1, "audio": "first", "transcription": "first"}, + {"_winml_source_index": 0, "id": 2, "audio": "second", "transcription": "second"}, ] evaluator._transcribe = MagicMock() @@ -437,11 +437,152 @@ def test_compute_rejects_missing_selected_source_index_before_inference() -> Non evaluator._transcribe.assert_not_called() +@pytest.mark.parametrize( + ("row", "message"), + [ + ( + { + "_winml_source_index": 0, + "audio": {"path": "sample.wav"}, + "transcription": "reference", + }, + "dataset ID", + ), + ( + { + "_winml_source_index": 0, + "id": True, + "audio": {"path": "sample.wav"}, + "transcription": "reference", + }, + "dataset ID", + ), + ( + { + "_winml_source_index": 0, + "id": " ", + "audio": {"path": "sample.wav"}, + "transcription": "reference", + }, + "dataset ID", + ), + ( + { + "_winml_source_index": 0, + "id": 1525, + "transcription": "reference", + }, + "audio identity", + ), + ( + { + "_winml_source_index": 0, + "id": 1525, + "audio": {"path": "", "key": ""}, + "transcription": "reference", + }, + "audio identity", + ), + ], +) +def test_compute_rejects_malformed_selected_identity_before_inference( + row: dict[str, object], + message: str, +) -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [row] + evaluator._transcribe = MagicMock() + + with pytest.raises(DatasetValidationError, match=message): + evaluator.compute() + + evaluator._transcribe.assert_not_called() + + +def test_compute_rejects_duplicate_audio_identity_before_inference() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + { + "_winml_source_index": 0, + "id": 1525, + "audio": {"path": r"C:\first\same.wav", "key": "first"}, + "transcription": "first", + }, + { + "_winml_source_index": 1, + "id": 1657, + "audio": {"path": r"D:\second\same.wav", "key": "second"}, + "transcription": "second", + }, + ] + evaluator._transcribe = MagicMock() + + with pytest.raises(DatasetValidationError, match="duplicate audio identities"): + evaluator.compute() + + evaluator._transcribe.assert_not_called() + + +def test_compute_accepts_duplicate_dataset_ids_with_distinct_row_and_audio_identity() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + { + "_winml_source_index": 0, + "id": 1525, + "audio": {"path": "first.wav"}, + "transcription": "same sentence", + }, + { + "_winml_source_index": 1, + "id": 1525, + "audio": {"path": "second.wav"}, + "transcription": "same sentence", + }, + ] + evaluator._transcribe = MagicMock(return_value="same sentence") + + result = evaluator.compute() + + assert result["selected_source_ids"] == [1525, 1525] + assert result["selected_source_indices"] == [0, 1] + assert result["selected_audio_paths"] == ["first.wav", "second.wav"] + assert evaluator._transcribe.call_count == 2 + + +def test_compute_accepts_and_emits_pinned_fleurs_row_identities() -> None: + evaluator = _evaluator(input_shape=[1, "samples"]) + evaluator.data = [ + { + "_winml_source_index": 0, + "id": 1525, + "audio": {"path": r"C:\cache\10018492969996036091.wav"}, + "transcription": "first", + }, + { + "_winml_source_index": 1, + "id": 1657, + "audio": {"path": r"C:\cache\10288018704489549018.wav"}, + "transcription": "second", + }, + ] + evaluator._transcribe = MagicMock(side_effect=["first", "second"]) + + result = evaluator.compute() + + assert result["selected_source_ids"] == [1525, 1657] + assert result["selected_source_indices"] == [0, 1] + assert result["selected_audio_paths"] == [ + "10018492969996036091.wav", + "10288018704489549018.wav", + ] + assert evaluator._transcribe.call_count == 2 + + def test_compute_scores_successful_empty_decode_as_deletions() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"_winml_source_index": 0, "audio": "empty", "transcription": "hello world"}, - {"_winml_source_index": 1, "audio": "decoded", "transcription": "good day"}, + {"_winml_source_index": 0, "id": 1, "audio": "empty", "transcription": "hello world"}, + {"_winml_source_index": 1, "id": 2, "audio": "decoded", "transcription": "good day"}, ] evaluator._transcribe = lambda value: "" if value == "empty" else "good day" @@ -459,8 +600,8 @@ def test_compute_scores_successful_empty_decode_as_deletions() -> None: def test_compute_scores_all_empty_hypotheses_as_deletions() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"_winml_source_index": 0, "audio": "first", "transcription": "hello world"}, - {"_winml_source_index": 1, "audio": "second", "transcription": "good day"}, + {"_winml_source_index": 0, "id": 1, "audio": "first", "transcription": "hello world"}, + {"_winml_source_index": 1, "id": 2, "audio": "second", "transcription": "good day"}, ] evaluator._transcribe = lambda _value: "" @@ -478,8 +619,18 @@ def test_compute_scores_all_empty_hypotheses_as_deletions() -> None: def test_compute_rejects_empty_reference_but_keeps_empty_hypothesis() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"_winml_source_index": 0, "audio": "invalid-reference", "transcription": " \t "}, - {"_winml_source_index": 1, "audio": "empty-hypothesis", "transcription": "valid"}, + { + "_winml_source_index": 0, + "id": 1, + "audio": "invalid-reference", + "transcription": " \t ", + }, + { + "_winml_source_index": 1, + "id": 2, + "audio": "empty-hypothesis", + "transcription": "valid", + }, ] evaluator._transcribe = lambda _value: "" @@ -498,8 +649,8 @@ def test_compute_rejects_empty_reference_but_keeps_empty_hypothesis() -> None: def test_compute_distinguishes_decode_failure_from_empty_decode() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) evaluator.data = [ - {"_winml_source_index": 0, "audio": "failure", "transcription": "rejected"}, - {"_winml_source_index": 1, "audio": "empty", "transcription": "kept"}, + {"_winml_source_index": 0, "id": 1, "audio": "failure", "transcription": "rejected"}, + {"_winml_source_index": 1, "id": 2, "audio": "empty", "transcription": "kept"}, ] def transcribe(value: str) -> str: @@ -521,14 +672,16 @@ def transcribe(value: str) -> str: def test_compute_fails_closed_when_all_rows_are_rejected() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) - evaluator.data = [{"_winml_source_index": 0, "audio": None, "transcription": ""}] + evaluator.data = [{"_winml_source_index": 0, "id": 1, "audio": "invalid", "transcription": ""}] with pytest.raises(DatasetValidationError, match="processed=0, rejected=1"): evaluator.compute() def test_unexpected_inference_error_propagates() -> None: evaluator = _evaluator(input_shape=[1, "samples"]) - evaluator.data = [{"_winml_source_index": 0, "audio": "value", "transcription": "reference"}] + evaluator.data = [ + {"_winml_source_index": 0, "id": 1, "audio": "value", "transcription": "reference"} + ] evaluator._transcribe = MagicMock(side_effect=RuntimeError("session failed")) with pytest.raises(RuntimeError, match="session failed"): evaluator.compute() From 4b528133fef8cbeed4e9a3aba9be42d22a3c0221 Mon Sep 17 00:00:00 2001 From: "Shiyi Zheng (from Dev Box)" Date: Mon, 24 Aug 2026 09:55:10 +0800 Subject: [PATCH 8/8] Honor seeded CTC evaluation shuffle --- src/winml/modelkit/eval/ctc_asr_evaluator.py | 15 +- tests/unit/eval/test_ctc_asr_evaluator.py | 215 ++++++++++++++++++- 2 files changed, 224 insertions(+), 6 deletions(-) diff --git a/src/winml/modelkit/eval/ctc_asr_evaluator.py b/src/winml/modelkit/eval/ctc_asr_evaluator.py index c23c9119d..19d0161b9 100644 --- a/src/winml/modelkit/eval/ctc_asr_evaluator.py +++ b/src/winml/modelkit/eval/ctc_asr_evaluator.py @@ -312,13 +312,20 @@ def prepare_data(self) -> Dataset: ) from error validate_dataset_columns(dataset, "automatic-speech-recognition", ds.columns_mapping) - if ds.streaming: + dataset = dataset.map( + lambda _row, index: {"_winml_source_index": index}, + with_indices=True, + ) + if ds.shuffle: + if not ds.streaming: + dataset = dataset.to_iterable_dataset() + dataset = dataset.shuffle(seed=ds.seed) + rows = list(dataset.take(ds.samples)) + elif ds.streaming: rows = list(dataset.take(ds.samples)) else: rows = [dataset[index] for index in range(min(ds.samples, len(dataset)))] - return Dataset.from_list( - [{**row, "_winml_source_index": index} for index, row in enumerate(rows)] - ) + return Dataset.from_list(rows) def compute(self) -> dict[str, Any]: """Decode bounded rows and report corpus WER/CER plus accounting.""" diff --git a/tests/unit/eval/test_ctc_asr_evaluator.py b/tests/unit/eval/test_ctc_asr_evaluator.py index 5056eebbb..863a27c4d 100644 --- a/tests/unit/eval/test_ctc_asr_evaluator.py +++ b/tests/unit/eval/test_ctc_asr_evaluator.py @@ -169,11 +169,17 @@ def test_prepare_data_preserves_source_order_with_duplicate_ids(streaming: bool) {"id": 1657, "audio": {"path": "second.wav"}, "transcription": "second"}, {"id": 1525, "audio": {"path": "third.wav"}, "transcription": "third"}, ] + indexed_rows = [{**row, "_winml_source_index": index} for index, row in enumerate(rows)] dataset = MagicMock(column_names=["id", "audio", "transcription"]) dataset.cast_column.return_value = dataset dataset.__len__.return_value = len(rows) - dataset.__getitem__.side_effect = rows.__getitem__ - dataset.take.return_value = rows + + def add_source_indices(*_args, **_kwargs): + dataset.__getitem__.side_effect = indexed_rows.__getitem__ + dataset.take.return_value = indexed_rows + return dataset + + dataset.map.side_effect = add_source_indices evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) evaluator._audio_column = "audio" evaluator.config = WinMLEvaluationConfig( @@ -190,6 +196,211 @@ def test_prepare_data_preserves_source_order_with_duplicate_ids(streaming: bool) dataset.sort.assert_not_called() +@pytest.mark.parametrize("streaming", [False, True]) +def test_prepare_data_applies_seeded_shuffle_before_bounded_selection(streaming: bool) -> None: + from datasets import Audio, Dataset + + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + + rows = [ + { + "id": index, + "audio": f"{index}.wav", + "transcription": str(index), + } + for index in range(10) + ] + dataset = Dataset.from_list(rows) + if streaming: + dataset = dataset.cast_column("audio", Audio(decode=False)).to_iterable_dataset() + evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) + evaluator._audio_column = "audio" + evaluator.config = WinMLEvaluationConfig( + task="automatic-speech-recognition", + dataset=DatasetConfig( + path="dataset", + samples=4, + shuffle=True, + seed=123, + streaming=streaming, + ), + ) + + with patch("datasets.load_dataset", return_value=dataset): + selected = evaluator.prepare_data() + + assert selected["id"] == [7, 4, 0, 2] + assert selected["_winml_source_index"] == [7, 4, 0, 2] + + +@pytest.mark.parametrize("streaming", [False, True]) +def test_prepare_data_shuffle_is_repeatable_and_seed_discriminating(streaming: bool) -> None: + from datasets import Dataset, Features, IterableDataset, Value + + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + + rows = [ + {"id": index, "audio": f"{index}.wav", "transcription": str(index)} for index in range(10) + ] + features = Features( + { + "id": Value("int64"), + "audio": Value("string"), + "transcription": Value("string"), + } + ) + + def select(seed: int) -> list[int]: + dataset = ( + IterableDataset.from_generator(lambda: iter(rows), features=features) + if streaming + else Dataset.from_list(rows) + ) + evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) + evaluator._audio_column = "audio" + evaluator.config = WinMLEvaluationConfig( + task="automatic-speech-recognition", + dataset=DatasetConfig( + path="dataset", + samples=4, + shuffle=True, + seed=seed, + streaming=streaming, + ), + ) + with ( + patch.object(dataset, "cast_column", return_value=dataset), + patch("datasets.load_dataset", return_value=dataset), + ): + return list(evaluator.prepare_data()["_winml_source_index"]) + + assert select(123) == select(123) == [7, 4, 0, 2] + assert select(456) != select(123) + + +def test_prepare_data_shuffle_modes_match_beyond_stream_buffer() -> None: + from datasets import Dataset, Features, IterableDataset, Value + + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + + rows = [ + {"id": index, "audio": f"{index}.wav", "transcription": str(index)} + for index in range(1_200) + ] + features = Features( + { + "id": Value("int64"), + "audio": Value("string"), + "transcription": Value("string"), + } + ) + + def select(streaming: bool) -> list[int]: + dataset = ( + IterableDataset.from_generator(lambda: iter(rows), features=features) + if streaming + else Dataset.from_list(rows) + ) + evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) + evaluator._audio_column = "audio" + evaluator.config = WinMLEvaluationConfig( + task="automatic-speech-recognition", + dataset=DatasetConfig( + path="dataset", + samples=4, + shuffle=True, + seed=123, + streaming=streaming, + ), + ) + with ( + patch.object(dataset, "cast_column", return_value=dataset), + patch("datasets.load_dataset", return_value=dataset), + ): + return list(evaluator.prepare_data()["_winml_source_index"]) + + assert select(streaming=True) == select(streaming=False) + + +def test_prepare_data_streaming_shuffle_is_bounded() -> None: + from datasets import Features, IterableDataset, Value + + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + + yielded = 0 + + def rows(): + nonlocal yielded + for index in range(10_000): + yielded += 1 + if yielded > 1_100: + raise AssertionError("streaming shuffle consumed an unbounded source") + yield {"id": index, "audio": f"{index}.wav", "transcription": str(index)} + + dataset = IterableDataset.from_generator( + rows, + features=Features( + { + "id": Value("int64"), + "audio": Value("string"), + "transcription": Value("string"), + } + ), + ) + evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) + evaluator._audio_column = "audio" + evaluator.config = WinMLEvaluationConfig( + task="automatic-speech-recognition", + dataset=DatasetConfig( + path="dataset", + samples=4, + shuffle=True, + seed=123, + streaming=True, + ), + ) + + with ( + patch.object(dataset, "cast_column", return_value=dataset), + patch("datasets.load_dataset", return_value=dataset), + ): + selected = evaluator.prepare_data() + + assert len(selected) == 4 + assert yielded <= 1_100 + assert all(0 <= index < yielded for index in selected["_winml_source_index"]) + + +def test_compute_rejects_malformed_identity_selected_by_shuffle_before_inference() -> None: + from datasets import Dataset + + from winml.modelkit.eval import DatasetConfig, WinMLEvaluationConfig + + rows = [ + { + "id": None if index == 7 else index, + "audio": f"{index}.wav", + "transcription": str(index), + } + for index in range(10) + ] + evaluator = WinMLCTCASREvaluator.__new__(WinMLCTCASREvaluator) + evaluator._audio_column = "audio" + evaluator._transcription_column = "transcription" + evaluator.config = WinMLEvaluationConfig( + task="automatic-speech-recognition", + dataset=DatasetConfig(path="dataset", samples=1, shuffle=True, seed=123), + ) + with patch("datasets.load_dataset", return_value=Dataset.from_list(rows)): + evaluator.data = evaluator.prepare_data() + evaluator._transcribe = MagicMock() + + assert evaluator.data["_winml_source_index"] == [7] + with pytest.raises(DatasetValidationError, match="dataset ID"): + evaluator.compute() + evaluator._transcribe.assert_not_called() + + def test_soundfile_decode_mixes_stereo_to_mono_float32() -> None: stereo = np.column_stack( [np.linspace(-1, 1, 32, dtype=np.float32), np.linspace(1, -1, 32, dtype=np.float32)]