Clarify Pi0.5 Thor benchmark warmup and latency gate

README.md

@@ -90,6 +90,9 @@ Baseline comparisons and source methodology live in [Benchmark Comparison](docs/
 | Jetson AGX Thor | NVFP4 + FA4, 3-view | **31.74 ms** | **32 Hz** | [Pi0.5 Thor NVFP4](docs/pi05_thor_decoder_fp4_e2e.md) |
 | RTX 5090 | FP8, 2-view | **17.58 ms** | **57 Hz** | [Blackwell VLA](examples/blackwell/README.md#vla-latency-rtx-5090) |
 
+For the Pi0.5 Thor rows, use about 300 warmup calls; 20 can be insufficient.
+See the [warmup and latency reference notes](docs/pi05_thor_decoder_fp4_e2e.md#warmup-and-latency-references).
+
 † At 1 view the per-sample action cosine against the FP8 reference does
 not clear the 0.995 gate (worst sample 0.971): with single-view input the
 flow-matching field itself is near a decision boundary on some samples,

docs/pi05_thor_decoder_fp4_e2e.md

@@ -559,10 +559,34 @@ classes directly and is required for the exploratory A/B knobs, and results
 from that mode are not public-API numbers. The recorded `result.json` carries
 the mode in `construction` and the exact call in `children.*.public_api_call`.
 
+### Warmup and latency references
+
+The harness defaults to 300 warmup calls per mode. On Thor, 20 calls can leave
+the system warming up during the timed run: a reproduced 2-view run moved from
+41.38 / 29.05 ms FP8 / FP4 p50 with 20 warmups to 38.62 / 27.25 ms with 300.
+Each child result also records ten ordered group medians in
+`latency_group_medians_ms`. A clear step between groups suggests the timing
+regime changed while samples were being collected.
+
+The latest README values and the performance gate serve different purposes:
+
+| Views | README reference | Earlier-tier regression baseline | Gate limit |
+|---:|---:|---:|---:|
+| 1 | 23.01 ms | 30.5 ms | 28.5 ms |
+| 2 | 27.17 ms | 36.3 ms | 34.3 ms |
+| 3 | 31.74 ms | 42.8 ms | 40.8 ms |
+
+The README column is the current comparison point. The regression baseline is
+retained only for the existing acceptance gate, which requires a 2 ms margin
+over that earlier tier. Passing the gate therefore does not claim that a run
+reproduced the latest README latency. These names replace the ambiguous
+`published_sota_p50_ms` fields and gate in earlier artifacts, so new
+`result.json` files carry `schema_version: 2`.
+
 Both processes run FA4: the comparison isolates NVFP4 against FP8 with the
 attention backend held fixed, so the reported speedup is not an FA4 speedup.
 
-The current multi-view run used:
+The 2026-08-05 multi-view run used:
 
 - NVIDIA Thor, compute capability 11.0, MAXN.
 - GPC min/max/current 1.575 GHz.
@@ -579,10 +603,10 @@ The current multi-view run used:
 
 The suite requires a clean tracked worktree and fails unless clocks and device
 identity match, all outputs are finite, FP4 is faster than FP8, and the
-per-view published-minus-2-ms p50 target passes. It also requires 2-view p95
-at most 40 ms and 3-view p50 at most 40 ms. Final 7D action cosine must be at
-least 0.999 globally and 0.995 for every sample; internal raw cosine must be at
-least 0.995 globally and for every sample.
+per-view earlier-tier regression baseline minus 2 ms limit passes. It also
+requires 2-view p95 at most 40 ms and 3-view p50 at most 40 ms. Final 7D action
+cosine must be at least 0.999 globally and 0.995 for every sample; internal raw
+cosine must be at least 0.995 globally and for every sample.
 
 ## Encoder Down v7 Multi-View Result (commit `8424808`, 2026-07-26)
 

examples/quickstart.py

@@ -50,7 +50,8 @@ def main():
     parser.add_argument('--benchmark', type=int, default=0)
     parser.add_argument('--warmup', type=int, default=500,
                         help="Warmup iters before timed run. RTX 5090 needs "
-                             "~500 to reach boost P-state; Thor can use 20.")
+                             "~500 to reach boost P-state; Pi0.5 on Thor "
+                             "may need ~300 for stable results.")
     parser.add_argument('--autotune', type=int, default=3,
                         help="Autotune trials: 0=off, 3=default, 5=thorough")
     parser.add_argument('--recalibrate', action='store_true',
@@ -130,7 +131,8 @@ def main():
     if args.benchmark > 0:
         # RTX 5090 takes ~500 replay iterations to climb from idle P8
         # (195 MHz) to boost P1 (~2870 MHz). Small warmup biases P50 by
-        # 2-3 ms. Jetson Thor settles faster, 50 is plenty there.
+        # 2-3 ms. Pi0.5 on Thor can also need ~300 iterations before the
+        # latency regime settles.
         warmup = args.warmup
         for _ in range(warmup):
             model.predict(images=imgs)

tests/bench_pi05_decoder_fp4_e2e.py

@@ -26,8 +26,12 @@
 
 EXPECTED_DEVICE = "NVIDIA Thor"
 EXPECTED_CC = (11, 0)
-PUBLISHED_SOTA_P50_MS = {1: 30.5, 2: 36.3, 3: 42.8}
-REQUIRED_MARGIN_MS = 2.0
+DEFAULT_WARMUP = 300
+LATENCY_GROUP_COUNT = 10
+RESULT_SCHEMA_VERSION = 2
+README_REFERENCE_P50_MS = {1: 23.01, 2: 27.17, 3: 31.74}
+REGRESSION_BASELINE_P50_MS = {1: 30.5, 2: 36.3, 3: 42.8}
+REQUIRED_REGRESSION_MARGIN_MS = 2.0
 TAIL_LATENCY_MS = 40.0
 PROMPT_TOKENS = [
     2, 18075, 908, 573, 3118, 3963, 578,
@@ -60,6 +64,17 @@
 }
 
 
+def latency_group_medians(samples_ms: list[float]) -> list[float]:
+    """Keep timing groups in run order so late warmup remains visible."""
+    if len(samples_ms) < LATENCY_GROUP_COUNT:
+        raise ValueError(
+            f"latency diagnostics require at least {LATENCY_GROUP_COUNT} samples")
+    return [
+        float(np.median(group))
+        for group in np.array_split(samples_ms, LATENCY_GROUP_COUNT)
+    ]
+
+
 def public_api_kwargs(mode: str, checkpoint: str, num_views: int) -> dict:
     """The exact load_model() call behind each published latency column."""
     kwargs = dict(
@@ -121,7 +136,10 @@ def main() -> int:
         "--fixture", help="exact N=8 fixture; defaults to the selected view count")
     parser.add_argument("--num-views", type=int, choices=(1, 2, 3), default=2)
     parser.add_argument("--output-dir")
-    parser.add_argument("--warmup", type=int, default=20)
+    parser.add_argument(
+        "--warmup", type=int, default=DEFAULT_WARMUP,
+        help="warmup calls per mode; approximately 300 are recommended for "
+             "stable Thor timing")
     parser.add_argument("--iters", type=int, default=100)
     parser.add_argument("--seed", type=int, default=20260725)
     parser.add_argument(
@@ -400,6 +418,7 @@ def main() -> int:
             "p95_ms": float(np.percentile(latencies, 95)),
             "min_ms": min(latencies),
             "max_ms": max(latencies),
+            "latency_group_medians_ms": latency_group_medians(latencies),
             "samples_ms": latencies,
         }
         print("__E2E_RESULT__ " + json.dumps(result, sort_keys=True), flush=True)
@@ -493,8 +512,9 @@ def main() -> int:
     fp8_p50 = float(child_results["fp8"]["p50_ms"])
     fp4_p50 = float(child_results["fp4"]["p50_ms"])
     fp4_p95 = float(child_results["fp4"]["p95_ms"])
-    published_sota = PUBLISHED_SOTA_P50_MS[args.num_views]
-    target_latency = published_sota - REQUIRED_MARGIN_MS
+    readme_reference = README_REFERENCE_P50_MS[args.num_views]
+    regression_baseline = REGRESSION_BASELINE_P50_MS[args.num_views]
+    regression_limit = regression_baseline - REQUIRED_REGRESSION_MARGIN_MS
     comparison = {
         "raw_cosine": float(
             raw_lhs @ raw_rhs /
@@ -512,9 +532,10 @@ def main() -> int:
         "p50_delta_ms": fp4_p50 - fp8_p50,
         "p50_speedup": fp8_p50 / fp4_p50,
         "num_views": args.num_views,
-        "published_sota_p50_ms": published_sota,
-        "required_margin_ms": REQUIRED_MARGIN_MS,
-        "target_p50_ms": target_latency,
+        "readme_reference_p50_ms": readme_reference,
+        "regression_baseline_p50_ms": regression_baseline,
+        "required_regression_margin_ms": REQUIRED_REGRESSION_MARGIN_MS,
+        "regression_limit_p50_ms": regression_limit,
     }
     gates = {
         "raw_cosine_at_least_0_995": comparison["raw_cosine"] >= 0.995,
@@ -524,13 +545,14 @@ def main() -> int:
         "action_min_sample_cosine_at_least_0_995": (
             comparison["action_min_sample_cosine"] >= 0.995),
         "fp4_p50_faster_than_fp8": fp4_p50 < fp8_p50,
-        "fp4_fa4_p50_beats_published_by_2ms": fp4_p50 <= target_latency,
+        "fp4_fa4_p50_within_regression_limit": fp4_p50 <= regression_limit,
     }
     if args.num_views == 2:
         gates["fp4_fa4_2v_p95_at_most_40_ms"] = fp4_p95 <= TAIL_LATENCY_MS
     if args.num_views == 3:
         gates["fp4_fa4_3v_p50_at_most_40_ms"] = fp4_p50 <= TAIL_LATENCY_MS
     result = {
+        "schema_version": RESULT_SCHEMA_VERSION,
         "timestamp_utc": datetime.now(timezone.utc).isoformat(),
         "commit": commit,
         "construction": args.construct,

tests/test_pi05_thor_fp4_routing.py

@@ -43,6 +43,30 @@ def _load_bench_module():
 PRESET = BENCH.PUBLIC_API_PRESET
 
 
+def test_benchmark_defaults_distinguish_reference_from_regression_gate():
+    assert BENCH.DEFAULT_WARMUP == 300
+    assert BENCH.RESULT_SCHEMA_VERSION == 2
+    assert BENCH.README_REFERENCE_P50_MS == {1: 23.01, 2: 27.17, 3: 31.74}
+    assert BENCH.REGRESSION_BASELINE_P50_MS == {1: 30.5, 2: 36.3, 3: 42.8}
+    assert BENCH.REQUIRED_REGRESSION_MARGIN_MS == 2.0
+
+
+def test_benchmark_latency_groups_preserve_measurement_order():
+    samples_ms = [
+        value
+        for group in range(10)
+        for value in (float(group), float(group + 2))
+    ]
+    assert BENCH.latency_group_medians(samples_ms) == [
+        float(group + 1) for group in range(10)
+    ]
+
+
+def test_benchmark_latency_groups_require_ten_samples():
+    with pytest.raises(ValueError, match="at least 10 samples"):
+        BENCH.latency_group_medians([1.0] * 9)
+
+
 # The constructor parameters the Pi0.5 Thor frontends expose. load_model
 # feature-detects with inspect.signature, so the stubs must declare them for
 # the routing to be exercised at all. test_stub_signatures_match_frontends