diff --git a/docs/commands/perf.md b/docs/commands/perf.md index 840d8b6b0..bd9bd2b4e 100644 --- a/docs/commands/perf.md +++ b/docs/commands/perf.md @@ -153,6 +153,10 @@ traced graph's inputs don't match the provided data — for example a compiled context model with different input names — the trace falls back to random inputs and logs a warning. +Op-tracing results are included in the main benchmark JSON under +`hw_monitor.ep_proof`. The profiling CSV remains available as the raw trace +artifact; no separate `_op_trace.json` file is written. + ## Common pitfalls - **Warm-up too low on NPU.** The first several inferences on an NPU EP can be significantly slower due to kernel compilation and caching. The default of 10 warm-up iterations is usually enough for vision models, but transformer models with many operators may need `--warmup 30` or higher to reach steady-state latency. diff --git a/scripts/e2e_eval/run_eval.py b/scripts/e2e_eval/run_eval.py index 1f38391d8..54e9d8312 100644 --- a/scripts/e2e_eval/run_eval.py +++ b/scripts/e2e_eval/run_eval.py @@ -1849,46 +1849,17 @@ def _resolve_op_tracing( return None -def _extract_op_trace_path(text: str) -> Path | None: - """Parse the ``Op-trace saved to: `` line from winml perf output. - - winml perf prints this via a Rich console, which hard-wraps the long path - across lines (inserting line breaks but no extra spaces) when stdout isn't a - TTY. Rejoin the wrapped fragments by dropping line breaks, then cut at the - ``.json`` terminator. Returns None when the line is absent. - """ - marker = "Op-trace saved to:" - idx = text.find(marker) - if idx == -1: - return None - tail = text[idx + len(marker) :].lstrip() - joined = tail.replace("\r", "").replace("\n", "") - end = joined.find(".json") - if end == -1: - return None - return Path(joined[: end + len(".json")]) - - def _copy_op_trace(proc: dict, output_path: Path, model_dir: Path, label: str = "") -> None: - """Copy the op-trace JSON produced by winml perf into ``model_dir``. - - The current perf contract writes the trace beside ``--output`` with an - ``_op_trace`` suffix. Console parsing remains as a fallback for older perf - versions. The destination is ``op_trace.json`` (suffixed with the sub-model - label for composite models). - """ - src = output_path.with_name(f"{output_path.stem}_op_trace{output_path.suffix}") - if not src.exists(): - src = _extract_op_trace_path(proc.get("stdout", "") + "\n" + proc.get("stderr", "")) - if src is None or not src.is_file(): + """Copy the perf JSON containing ``hw_monitor.ep_proof`` into ``model_dir``.""" + if proc.get("result") is None or not output_path.is_file(): return dest_name = f"op_trace_{label}.json" if label else "op_trace.json" dest = model_dir / dest_name try: - shutil.copyfile(src, dest) + shutil.copyfile(output_path, dest) safe_print(f" op-tracing: {dest}") except OSError as e: - safe_print(f" op-tracing: failed to copy {src} -> {dest}: {e}") + safe_print(f" op-tracing: failed to copy {output_path} -> {dest}: {e}") def _run_structured_perf( @@ -1930,8 +1901,9 @@ def run_model( summed elapsed). Multi-model structured results are keyed by sub-model label. When op_tracing is set, ``--op-tracing `` is passed to winml perf. The - op-trace JSON beside the structured perf output is copied into ``model_dir`` - as ``op_trace.json`` (suffixed with the sub-model label for composite models). + structured perf output containing ``hw_monitor.ep_proof`` is copied into + ``model_dir`` as ``op_trace.json`` (suffixed with the sub-model label for + composite models). """ trace = bool(op_tracing) and model_dir is not None @@ -2833,8 +2805,9 @@ def parse_args() -> argparse.Namespace: "defaults to 'basic' for a model whose 'op_tracing_targets' includes the " "current _ target (e.g. QNNExecutionProvider_npu) — this " "auto-enable requires both --ep and --device to be set explicitly " - "(the default --device auto does not match). The " - "resulting op_trace.json is copied into each model's output folder." + "(the default --device auto does not match). The resulting perf JSON, " + "including its embedded op trace, is copied into each model's output " + "folder as op_trace.json." ), ) parser.add_argument( diff --git a/src/winml/modelkit/commands/perf.py b/src/winml/modelkit/commands/perf.py index 64151fb81..bd3ca91ac 100644 --- a/src/winml/modelkit/commands/perf.py +++ b/src/winml/modelkit/commands/perf.py @@ -105,6 +105,10 @@ def _detail_fallback_guidance(reason: TraceFallbackReason | None) -> str: ), TraceFallbackReason.QHAS_OUTPUT_MISSING: "the requested QHAS output was not found", TraceFallbackReason.QHAS_PARSE_FAILED: "the QHAS output could not be parsed", + TraceFallbackReason.MULTIPLE_PARTITIONS: ( + "the model contains multiple EPContext partitions; QHAS currently " + "reports only one partition" + ), } if reason is None: return "QHAS post-processing was unavailable" @@ -2144,17 +2148,9 @@ def _io_specs_from_config( def _print_save_to_footer( console: Console, *, - trace_json: str | None, profiling_csv: str | None, ) -> None: - """Print save-to footer lines after the op-trace report. - - Each line is rendered only when its path is supplied; if both are - ``None`` the helper emits nothing. The ``[dim]...[/dim]`` markup - softens the label so the path itself is the visual anchor. - """ - if trace_json: - console.print(f"[dim]Op-trace JSON:[/dim] {trace_json}") + """Print the raw profiling artifact path after the op-trace report.""" if profiling_csv: console.print(f"[dim]Profiling CSV:[/dim] {profiling_csv}") @@ -3399,7 +3395,7 @@ def perf( # misleading JSON artifact on disk for CI consumers. # ================================================================= if op_tracing: - from ..session.monitor.report import display_op_trace_report, write_op_trace_json + from ..session.monitor.report import display_op_trace_report # Both ONNX and HF inputs run through the same PerfBenchmark # instance, which exposes its perf context as ``_perf_ctx``. @@ -3464,16 +3460,9 @@ def perf( display_op_trace_report(trace_result, console, top_n=top_k) else: display_op_trace_report(trace_result, console) - # Write the op-trace report next to the requested benchmark output - # file (same directory + stem, with an ``_op_trace`` suffix) so the - # two artifacts stay paired instead of landing under an unrelated - # fixed name. - trace_output = output.with_name(f"{output.stem}_op_trace{output.suffix}") - write_op_trace_json(trace_result, trace_output) profiling_csv = trace_result.artifacts.get("csv") _print_save_to_footer( console, - trace_json=str(trace_output), profiling_csv=profiling_csv, ) else: diff --git a/src/winml/modelkit/session/monitor/op_metrics.py b/src/winml/modelkit/session/monitor/op_metrics.py index 82365323b..b0134bbac 100644 --- a/src/winml/modelkit/session/monitor/op_metrics.py +++ b/src/winml/modelkit/session/monitor/op_metrics.py @@ -44,6 +44,7 @@ class TraceFallbackReason(StrEnum): SCHEMATIC_PUBLISH_FAILED = "schematic_publish_failed" QHAS_OUTPUT_MISSING = "qhas_output_missing" QHAS_PARSE_FAILED = "qhas_parse_failed" + MULTIPLE_PARTITIONS = "multiple_partitions" @dataclass diff --git a/src/winml/modelkit/session/monitor/qnn_monitor.py b/src/winml/modelkit/session/monitor/qnn_monitor.py index c6f8f8f7d..a4385d06f 100644 --- a/src/winml/modelkit/session/monitor/qnn_monitor.py +++ b/src/winml/modelkit/session/monitor/qnn_monitor.py @@ -29,7 +29,7 @@ from typing import TYPE_CHECKING, Any, ClassVar, Literal from ..._env import env_flag_enabled -from ...onnx.epcontext import select_main_epcontext_partition_name +from ...onnx.epcontext import epcontext_partitions, select_main_epcontext_partition_name from ._onnx_metadata import _load_onnx_operator_data from .ep_monitor import WinMLEPMonitor from .op_metrics import ( @@ -58,6 +58,69 @@ } +def _metadata_int(value: object, field: str) -> int: + """Normalize a QNN numeric metadata value, including float strings.""" + try: + return round(float(value)) # type: ignore[arg-type] + except (TypeError, ValueError): + logger.warning( + "QNNMonitor: could not parse %r as a number for metadata field %r; " + "defaulting to 0. This may corrupt cycle_to_us and duration_us values.", + value, + field, + ) + return 0 + + +def _coalesce_partition_samples( + samples: list[dict[str, Any]], + partition_count: int, +) -> list[dict[str, Any]]: + """Combine the per-partition profiling blocks emitted for each inference.""" + if partition_count == 1: + return samples + + combined_samples: list[dict[str, Any]] = [] + for offset in range(0, len(samples), partition_count): + partition_samples = samples[offset : offset + partition_count] + combined: dict[str, Any] = { + "metadata": { + "hvx_threads": max( + _metadata_int(sample["metadata"]["hvx_threads"], "hvx_threads") + for sample in partition_samples + ), + "accel_execute_cycles": sum( + _metadata_int( + sample["metadata"]["accel_execute_cycles"], + "accel_execute_cycles", + ) + for sample in partition_samples + ), + "accel_execute_us": sum( + _metadata_int( + sample["metadata"]["accel_execute_us"], + "accel_execute_us", + ) + for sample in partition_samples + ), + }, + "samples": [], + } + for partition_index, sample in enumerate(partition_samples): + metadata = sample["metadata"] + for op in sample["samples"]: + combined["samples"].append( + { + **op, + "_partition_index": partition_index, + "_accel_execute_cycles": metadata["accel_execute_cycles"], + "_accel_execute_us": metadata["accel_execute_us"], + } + ) + combined_samples.append(combined) + return combined_samples + + class QNNMonitor(WinMLEPMonitor): """Qualcomm NPU per-op profiler via ORT's QNN EP. @@ -521,48 +584,47 @@ def _parse_artifacts(self, qhas_override: Path | None = None) -> OpTraceResult: parsed = parse_qnn_profiling_csv(csv_path) samples = parsed.get("samples", []) + partition_count = self._epcontext_partition_count() if self._expected_measured_samples is not None: expected_total = self._warmup_samples + self._expected_measured_samples - if len(samples) != expected_total: + expected_profile_blocks = expected_total * partition_count + if ( + len(samples) != expected_profile_blocks + and partition_count == 1 + and expected_total > 0 + and len(samples) % expected_total == 0 + ): + partition_count = len(samples) // expected_total + expected_profile_blocks = len(samples) + if len(samples) != expected_profile_blocks: raise ValueError( "profiling CSV sample count mismatch: " - f"expected {expected_total} total samples " + f"expected {expected_profile_blocks} profiling blocks " f"({self._warmup_samples} warmup + " - f"{self._expected_measured_samples} measured), got {len(samples)}" + f"{self._expected_measured_samples} measured) across " + f"{partition_count} EPContext partition(s), got {len(samples)}" ) + samples = _coalesce_partition_samples(samples, partition_count) samples = samples[self._warmup_samples :] artifacts: dict[str, str] = {"csv": str(csv_path)} - # Convert cycles to microseconds via the CSV-reported ratio. - # Use round(float(...)) rather than int() so that a float-string - # value like "12345.6" (legal QNN SDK output) parses correctly - # instead of raising ValueError → silent op-record drop. - def _to_int(val: object, field: str) -> int: - try: - return round(float(val)) # type: ignore[arg-type] - except (TypeError, ValueError): - logger.warning( - "QNNMonitor: could not parse %r as a number for metadata field %r; " - "defaulting to 0. This may corrupt cycle_to_us and duration_us values.", - val, - field, - ) - return 0 - sample_metadata: list[dict[str, int]] = [] - operator_samples: dict[int, dict[str, Any]] = {} + operator_samples: dict[tuple[int, int], dict[str, Any]] = {} for sample in samples: sample_meta = sample.get("metadata", {}) - total_cycles = _to_int( + total_cycles = _metadata_int( sample_meta.get("accel_execute_cycles", 0) or 0, "accel_execute_cycles", ) - accel_us = _to_int( + accel_us = _metadata_int( sample_meta.get("accel_execute_us", 0) or 0, "accel_execute_us", ) - hvx_threads = _to_int(sample_meta.get("hvx_threads", 0) or 0, "hvx_threads") + hvx_threads = _metadata_int( + sample_meta.get("hvx_threads", 0) or 0, + "hvx_threads", + ) sample_metadata.append( { "hvx_threads": hvx_threads, @@ -570,11 +632,12 @@ def _to_int(val: object, field: str) -> int: "accel_execute_us": accel_us, } ) - cycle_to_us = accel_us / total_cycles if total_cycles > 0 else 0.0 for op in sample.get("samples", []): op_id = op["op_id"] + partition_index = op.get("_partition_index", 0) + key = (partition_index, op_id) entry = operator_samples.setdefault( - op_id, + key, { "op_path": op["op_path"], "op_id": op_id, @@ -582,6 +645,17 @@ def _to_int(val: object, field: str) -> int: "percentages": [], }, ) + op_total_cycles = _metadata_int( + op.get("_accel_execute_cycles", total_cycles), + "accel_execute_cycles", + ) + op_accel_us = _metadata_int( + op.get("_accel_execute_us", accel_us), + "accel_execute_us", + ) + cycle_to_us = ( + op_accel_us / op_total_cycles if op_total_cycles > 0 else 0.0 + ) entry["durations_us"].append(op["cycles"] * cycle_to_us) entry["percentages"].append( op["cycles"] / total_cycles * 100 if total_cycles > 0 else 0.0 @@ -638,17 +712,25 @@ def _metadata_mean(field: str) -> float: fallback_reason: TraceFallbackReason | None = None # Detail mode: attempt QHAS post-processing. if self._level == "detail": - qhas_summary, qhas_operators, qhas_path, fallback_reason = self._try_qhas( - artifacts, qhas_override=qhas_override - ) - if qhas_path is not None and qhas_operators is not None: - operators = qhas_operators - summary = qhas_summary or summary - artifacts["qhas"] = str(qhas_path) - else: - # Fell back to CSV-only data in detail mode. + if partition_count > 1: status = "basic_fallback" - logger.warning("QNNMonitor: QHAS unavailable; detail mode degraded to basic") + fallback_reason = TraceFallbackReason.MULTIPLE_PARTITIONS + logger.warning( + "QNNMonitor: detail tracing has multiple EPContext partitions; " + "using the complete basic CSV trace" + ) + else: + qhas_summary, qhas_operators, qhas_path, fallback_reason = self._try_qhas( + artifacts, qhas_override=qhas_override + ) + if qhas_path is not None and qhas_operators is not None: + operators = qhas_operators + summary = qhas_summary or summary + artifacts["qhas"] = str(qhas_path) + else: + # Fell back to CSV-only data in detail mode. + status = "basic_fallback" + logger.warning("QNNMonitor: QHAS unavailable; detail mode degraded to basic") return OpTraceResult( model=None, @@ -664,6 +746,20 @@ def _metadata_mean(field: str) -> float: fallback_reason=fallback_reason, ) + def _epcontext_partition_count(self) -> int: + """Return the number of EPContext partitions represented in each inference.""" + if self._running_model_path is None: + return 1 + try: + return max(1, len(epcontext_partitions(self._running_model_path))) + except Exception as exc: + logger.info( + "QNNMonitor: unable to inspect EPContext partitions from %s: %s", + self._running_model_path, + exc, + ) + return 1 + def _try_qhas( self, artifacts: dict[str, str], diff --git a/tests/e2e/test_perf_e2e.py b/tests/e2e/test_perf_e2e.py index 83a8067b5..82b518739 100644 --- a/tests/e2e/test_perf_e2e.py +++ b/tests/e2e/test_perf_e2e.py @@ -820,8 +820,9 @@ def test_op_tracing_basic_qnn_npu(self, tmp_path: Path, npu_model_arg: str): assert result.exit_code == 0, f"perf failed (exit {result.exit_code}):\n{result.output}" assert output_file.exists() - assert trace_output.exists() - trace = json.loads(trace_output.read_text()) + assert not trace_output.exists() + output = json.loads(output_file.read_text()) + trace = output["hw_monitor"]["ep_proof"] assert trace["metadata"]["device"] == "npu" assert trace["metadata"]["ep"] == EP_ALIASES["qnn"] assert trace["metadata"]["tracing_level"] == "basic" diff --git a/tests/unit/commands/test_perf_optracing.py b/tests/unit/commands/test_perf_optracing.py index e6084db91..ba332bdce 100644 --- a/tests/unit/commands/test_perf_optracing.py +++ b/tests/unit/commands/test_perf_optracing.py @@ -921,7 +921,6 @@ def _invoke_text_op_trace_failure(tmp_path: Path, trace_result): patch("winml.modelkit.commands.perf.display_console_report") as display_report, patch("winml.modelkit.commands.perf.write_json_report") as write_json, patch("winml.modelkit.session.monitor.report.display_op_trace_report") as display_trace, - patch("winml.modelkit.session.monitor.report.write_op_trace_json") as write_trace, ): result = runner.invoke( perf, @@ -938,7 +937,7 @@ def _invoke_text_op_trace_failure(tmp_path: Path, trace_result): obj={}, ) - return result, display_report, write_json, display_trace, write_trace + return result, display_report, write_json, display_trace class TestCliOpTracingDispatch: @@ -982,7 +981,7 @@ def test_text_mode_not_run_status_exits_before_success_reports(self, tmp_path: P status="not_run", ) - result, display_report, write_json, display_trace, write_trace = ( + result, display_report, write_json, display_trace = ( _invoke_text_op_trace_failure(tmp_path, trace) ) @@ -991,7 +990,6 @@ def test_text_mode_not_run_status_exits_before_success_reports(self, tmp_path: P display_report.assert_not_called() write_json.assert_not_called() display_trace.assert_not_called() - write_trace.assert_not_called() def test_json_mode_missing_trace_result_does_not_emit_benchmark_json( self, tmp_path: Path @@ -1143,7 +1141,6 @@ def test_basic_fallback_status_exits_0_with_notice(self, tmp_path: Path): patch("winml.modelkit.commands.perf.display_console_report"), patch("winml.modelkit.commands.perf.write_json_report"), patch("winml.modelkit.session.monitor.report.display_op_trace_report"), - patch("winml.modelkit.session.monitor.report.write_op_trace_json"), patch("winml.modelkit.onnx.is_compiled_onnx", return_value=True), ): result = runner.invoke( @@ -1223,7 +1220,6 @@ def test_basic_fallback_status_rejects_raw_running_model(self, tmp_path: Path): patch("winml.modelkit.commands.perf.display_console_report"), patch("winml.modelkit.commands.perf.write_json_report"), patch("winml.modelkit.session.monitor.report.display_op_trace_report"), - patch("winml.modelkit.session.monitor.report.write_op_trace_json"), patch("winml.modelkit.onnx.is_compiled_onnx", return_value=False), ): result = runner.invoke( @@ -1317,7 +1313,7 @@ def _capture_write(*args, **kwargs): # PRD §10.5 / coreloop §8.4 mandate this test: # "test_cli_op_tracing_basic_on_qnn (skip if no QNN NPU): runs # wmk perf -m resnet50 --device npu --op-tracing basic, asserts CSV -# produced, *_op_trace.json written, at least one operator entry." +# produced, op trace embedded in the perf JSON, at least one operator entry." # # This is the only end-to-end proof that SC-1 holds: the headline # invocation produces real per-operator trace data on a QNN NPU. @@ -1339,7 +1335,7 @@ def test_cli_op_tracing_basic_on_qnn(tmp_path): Hardware-gated. Must produce: * a profiling CSV under the monitor's output directory, - * a ``*_op_trace.json`` next to the perf JSON output, + * op-trace data embedded in the perf JSON output, * at least one operator entry, with ``status == "ok"``. A regression that silently falls back to CPU (the bug SC-1 explicitly @@ -1378,15 +1374,11 @@ def test_cli_op_tracing_basic_on_qnn(tmp_path): f"perf --op-tracing basic failed (exit {result.exit_code}):\n{result.output}" ) - # Per-op trace JSON written next to the perf output. - trace_files = list(tmp_path.glob("*_op_trace.json")) - assert trace_files, ( - f"Expected *_op_trace.json next to {output_path}; got: {list(tmp_path.iterdir())}" - ) - import json - trace_data = json.loads(trace_files[0].read_text(encoding="utf-8")) + report_data = json.loads(output_path.read_text(encoding="utf-8")) + trace_data = report_data["hw_monitor"]["ep_proof"] + assert not list(tmp_path.glob("*_op_trace.json")) assert trace_data["status"] == "ok", ( f"Expected status='ok' on real hardware, got {trace_data['status']!r} " f"with error={trace_data.get('error')!r}" diff --git a/tests/unit/commands/test_perf_save_footer.py b/tests/unit/commands/test_perf_save_footer.py index 0dcaff07d..f115e8aa7 100644 --- a/tests/unit/commands/test_perf_save_footer.py +++ b/tests/unit/commands/test_perf_save_footer.py @@ -13,38 +13,23 @@ from winml.modelkit.commands.perf import _print_save_to_footer -def _render(trace_json: str | None, profiling_csv: str | None) -> str: +def _render(profiling_csv: str | None) -> str: buf = StringIO() console = Console(file=buf, width=120, force_terminal=False, record=True) - _print_save_to_footer(console, trace_json=trace_json, profiling_csv=profiling_csv) + _print_save_to_footer(console, profiling_csv=profiling_csv) return console.export_text() -def test_both_paths_shown(): - out = _render(r"C:\out\trace.json", r"C:\out\prof.csv") - assert "trace.json" in out +def test_csv_path_shown(): + out = _render(r"C:\out\prof.csv") assert "prof.csv" in out -def test_csv_omitted_when_none(): - out = _render(r"C:\out\trace.json", None) - assert "trace.json" in out - assert ".csv" not in out - - -def test_neither_when_both_none(): - out = _render(None, None) +def test_footer_omitted_when_csv_is_none(): + out = _render(None) assert out.strip() == "" -def test_json_path_label_present(): - out = _render(r"C:\out\trace.json", None) - # The footer should label what each path is. Look for "Op-trace JSON" or - # similar marker so users know what the path means. - assert "Op-trace" in out or "trace JSON" in out.lower() - - def test_csv_path_label_present(): - out = _render(r"C:\out\trace.json", r"C:\out\prof.csv") - # Similarly the CSV line should be labeled. + out = _render(r"C:\out\prof.csv") assert "CSV" in out or "csv" in out.lower() diff --git a/tests/unit/eval/test_run_eval_script.py b/tests/unit/eval/test_run_eval_script.py index e2a363d2c..aed238e41 100644 --- a/tests/unit/eval/test_run_eval_script.py +++ b/tests/unit/eval/test_run_eval_script.py @@ -1368,16 +1368,20 @@ def fake_run(args, timeout): assert proc["result"] is None assert "Invalid structured winml perf output" in proc["stderr"] - def test_op_trace_sibling_is_copied_before_cleanup(self, run_eval, tmp_path): + def test_perf_result_with_embedded_op_trace_is_copied_before_cleanup( + self, run_eval, tmp_path + ): trace_result = {"status": "ok", "operators": []} + perf_result = { + **_perf_result(), + "hw_monitor": {"ep_proof": trace_result}, + } def fake_run(args, timeout): output_path = Path(args[args.index("--output") + 1]) - output_path.write_text(json.dumps(_perf_result()), encoding="utf-8") - trace_path = output_path.with_name(f"{output_path.stem}_op_trace{output_path.suffix}") - trace_path.write_text(json.dumps(trace_result), encoding="utf-8") + output_path.write_text(json.dumps(perf_result), encoding="utf-8") return { - "stdout": f"Op-trace JSON: {trace_path}", + "stdout": "", "stderr": "", "exit_code": 0, "elapsed": 1.0, @@ -1395,7 +1399,7 @@ def fake_run(args, timeout): model_dir=tmp_path, ) - assert json.loads((tmp_path / "op_trace.json").read_text(encoding="utf-8")) == trace_result + assert json.loads((tmp_path / "op_trace.json").read_text(encoding="utf-8")) == perf_result def test_missing_structured_output_fails_perf(self, run_eval, tmp_path): proc_result = { diff --git a/tests/unit/session/monitor/test_qnn_monitor.py b/tests/unit/session/monitor/test_qnn_monitor.py index b01b478a9..6fa86c0f9 100644 --- a/tests/unit/session/monitor/test_qnn_monitor.py +++ b/tests/unit/session/monitor/test_qnn_monitor.py @@ -68,7 +68,7 @@ def _write_basic_profile( "CYCLES", "BACKEND", "SUB-EVENT", - f"{operator_name}:OpId_1 (cycles)", + f"{sample.get('operator_name', operator_name)}:OpId_1 (cycles)", ] ) writer.writerow( @@ -485,6 +485,191 @@ def test_basic_metrics_exclude_warmup_samples(tmp_path): ) +def test_basic_metrics_coalesce_multiple_epcontext_partitions(tmp_path): + from winml.modelkit.session.monitor.qnn_monitor import QNNMonitor + + context_model = tmp_path / "model_ctx.onnx" + _write_epcontext_model( + context_model, + [("first_partition", 1), ("second_partition", 0)], + ) + profile_blocks = [ + { + "hvx_threads": 4, + "accel_execute_cycles": 100, + "accel_execute_us": 10, + "operator_cycles": 50, + "operator_name": "FirstOp", + }, + { + "hvx_threads": 4, + "accel_execute_cycles": 200, + "accel_execute_us": 40, + "operator_cycles": 100, + "operator_name": "SecondOp", + }, + { + "hvx_threads": 4, + "accel_execute_cycles": 200, + "accel_execute_us": 40, + "operator_cycles": 100, + "operator_name": "FirstOp", + }, + { + "hvx_threads": 4, + "accel_execute_cycles": 400, + "accel_execute_us": 120, + "operator_cycles": 200, + "operator_name": "SecondOp", + }, + ] + monitor = QNNMonitor(output_dir=tmp_path) + monitor.set_running_model_path(context_model) + monitor.set_perf_window(warmup=0, measured_iterations=2) + monitor.__enter__() + _write_basic_profile(monitor._csv_path, profile_blocks) + monitor.__exit__(None, None, None) + + assert monitor.result is not None + assert monitor.result.status == "ok" + assert monitor.result.num_samples == 2 + operators = {operator.op_path: operator for operator in monitor.result.operators} + assert operators["FirstOp"].samples_us == [5.0, 20.0] + assert operators["SecondOp"].samples_us == [20.0, 60.0] + assert monitor.result.summary["accel_execute_cycles"] == 450 + assert monitor.result.summary["accel_execute_us"] == 105 + + +def test_basic_metrics_coalesce_float_string_partition_metadata(tmp_path, monkeypatch): + from winml.modelkit.session.monitor import qnn_monitor as qnn_mod + from winml.modelkit.session.monitor.qnn_monitor import QNNMonitor + + context_model = tmp_path / "model_ctx.onnx" + _write_epcontext_model( + context_model, + [("first_partition", 1), ("second_partition", 0)], + ) + parsed = { + "samples": [ + { + "metadata": { + "hvx_threads": "4.0", + "accel_execute_cycles": "100.4", + "accel_execute_us": "10.4", + }, + "samples": [{"op_path": "FirstOp", "op_id": 1, "cycles": 50}], + }, + { + "metadata": { + "hvx_threads": "4.0", + "accel_execute_cycles": "200.6", + "accel_execute_us": "40.6", + }, + "samples": [{"op_path": "SecondOp", "op_id": 1, "cycles": 100}], + }, + ] + } + monitor = QNNMonitor(output_dir=tmp_path) + monitor.set_running_model_path(context_model) + monitor.set_perf_window(warmup=0, measured_iterations=1) + monitor.__enter__() + monitor._csv_path.write_text("profile", encoding="utf-8") + monkeypatch.setattr(qnn_mod, "parse_qnn_profiling_csv", lambda _path: parsed) + + monitor.__exit__(None, None, None) + + assert monitor.result is not None + assert monitor.result.status == "ok" + assert monitor.result.num_samples == 1 + assert monitor.result.summary["accel_execute_cycles"] == 301 + assert monitor.result.summary["accel_execute_us"] == 51 + operators = {operator.op_path: operator for operator in monitor.result.operators} + assert operators["FirstOp"].samples_us == [5.0] + assert operators["SecondOp"].samples_us == pytest.approx([100 * 41 / 201]) + + +def test_basic_metrics_infer_runtime_partitions_for_raw_model(tmp_path): + from winml.modelkit.session.monitor.qnn_monitor import QNNMonitor + + raw_model = tmp_path / "model.onnx" + _write_transpose_model(raw_model) + profile_blocks = [ + { + "hvx_threads": 4, + "accel_execute_cycles": cycles, + "accel_execute_us": duration, + "operator_cycles": cycles // 2, + "operator_name": operator_name, + } + for cycles, duration, operator_name in [ + (100, 10, "FirstOp"), + (200, 40, "SecondOp"), + (200, 40, "FirstOp"), + (400, 120, "SecondOp"), + ] + ] + monitor = QNNMonitor(output_dir=tmp_path) + monitor.set_running_model_path(raw_model) + monitor.set_perf_window(warmup=0, measured_iterations=2) + monitor.__enter__() + _write_basic_profile(monitor._csv_path, profile_blocks) + monitor.__exit__(None, None, None) + + assert monitor.result is not None + assert monitor.result.status == "ok" + assert monitor.result.num_samples == 2 + operators = {operator.op_path: operator for operator in monitor.result.operators} + assert operators["FirstOp"].samples_us == [5.0, 20.0] + assert operators["SecondOp"].samples_us == [20.0, 60.0] + + +def test_detail_metrics_fall_back_to_complete_csv_for_multiple_partitions( + tmp_path, monkeypatch +): + from winml.modelkit.session.monitor.qnn_monitor import QNNMonitor + + context_model = tmp_path / "model_ctx.onnx" + _write_epcontext_model( + context_model, + [("first_partition", 1), ("second_partition", 0)], + ) + profile_blocks = [ + { + "hvx_threads": 4, + "accel_execute_cycles": 100, + "accel_execute_us": 10, + "operator_cycles": 50, + "operator_name": "FirstOp", + }, + { + "hvx_threads": 4, + "accel_execute_cycles": 200, + "accel_execute_us": 40, + "operator_cycles": 100, + "operator_name": "SecondOp", + }, + ] + monitor = QNNMonitor(level="detail", output_dir=tmp_path) + monitor.set_running_model_path(context_model) + monitor.set_perf_window(warmup=0, measured_iterations=1) + monitor.__enter__() + _write_basic_profile(monitor._csv_path, profile_blocks) + try_qhas = MagicMock() + monkeypatch.setattr(monitor, "_try_qhas", try_qhas) + + monitor.__exit__(None, None, None) + + assert monitor.result is not None + assert monitor.result.status == "basic_fallback" + assert monitor.result.fallback_reason == TraceFallbackReason.MULTIPLE_PARTITIONS + assert {operator.op_path for operator in monitor.result.operators} == { + "FirstOp", + "SecondOp", + } + assert "qhas" not in monitor.result.artifacts + try_qhas.assert_not_called() + + def test_basic_metrics_omit_onnx_metadata_when_env_disabled(tmp_path, monkeypatch): from winml.modelkit.session.monitor.qnn_monitor import QNNMonitor