diff --git a/README.md b/README.md index 994e3b53..7852a876 100644 --- a/README.md +++ b/README.md @@ -90,6 +90,9 @@ Baseline comparisons and source methodology live in [Benchmark Comparison](docs/ | Jetson AGX Thor | NVFP4 + FA4, 3-view | **31.74 ms** | **32 Hz** | [Pi0.5 Thor NVFP4](docs/pi05_thor_decoder_fp4_e2e.md) | | RTX 5090 | FP8, 2-view | **17.58 ms** | **57 Hz** | [Blackwell VLA](examples/blackwell/README.md#vla-latency-rtx-5090) | +For the Pi0.5 Thor rows, use about 300 warmup calls; 20 can be insufficient. +See the [warmup and latency reference notes](docs/pi05_thor_decoder_fp4_e2e.md#warmup-and-latency-references). + † At 1 view the per-sample action cosine against the FP8 reference does not clear the 0.995 gate (worst sample 0.971): with single-view input the flow-matching field itself is near a decision boundary on some samples, diff --git a/docs/pi05_thor_decoder_fp4_e2e.md b/docs/pi05_thor_decoder_fp4_e2e.md index b8751d27..1ddfd3b2 100644 --- a/docs/pi05_thor_decoder_fp4_e2e.md +++ b/docs/pi05_thor_decoder_fp4_e2e.md @@ -559,10 +559,34 @@ classes directly and is required for the exploratory A/B knobs, and results from that mode are not public-API numbers. The recorded `result.json` carries the mode in `construction` and the exact call in `children.*.public_api_call`. +### Warmup and latency references + +The harness defaults to 300 warmup calls per mode. On Thor, 20 calls can leave +the system warming up during the timed run: a reproduced 2-view run moved from +41.38 / 29.05 ms FP8 / FP4 p50 with 20 warmups to 38.62 / 27.25 ms with 300. +Each child result also records ten ordered group medians in +`latency_group_medians_ms`. A clear step between groups suggests the timing +regime changed while samples were being collected. + +The latest README values and the performance gate serve different purposes: + +| Views | README reference | Earlier-tier regression baseline | Gate limit | +|---:|---:|---:|---:| +| 1 | 23.01 ms | 30.5 ms | 28.5 ms | +| 2 | 27.17 ms | 36.3 ms | 34.3 ms | +| 3 | 31.74 ms | 42.8 ms | 40.8 ms | + +The README column is the current comparison point. The regression baseline is +retained only for the existing acceptance gate, which requires a 2 ms margin +over that earlier tier. Passing the gate therefore does not claim that a run +reproduced the latest README latency. These names replace the ambiguous +`published_sota_p50_ms` fields and gate in earlier artifacts, so new +`result.json` files carry `schema_version: 2`. + Both processes run FA4: the comparison isolates NVFP4 against FP8 with the attention backend held fixed, so the reported speedup is not an FA4 speedup. -The current multi-view run used: +The 2026-08-05 multi-view run used: - NVIDIA Thor, compute capability 11.0, MAXN. - GPC min/max/current 1.575 GHz. @@ -579,10 +603,10 @@ The current multi-view run used: The suite requires a clean tracked worktree and fails unless clocks and device identity match, all outputs are finite, FP4 is faster than FP8, and the -per-view published-minus-2-ms p50 target passes. It also requires 2-view p95 -at most 40 ms and 3-view p50 at most 40 ms. Final 7D action cosine must be at -least 0.999 globally and 0.995 for every sample; internal raw cosine must be at -least 0.995 globally and for every sample. +per-view earlier-tier regression baseline minus 2 ms limit passes. It also +requires 2-view p95 at most 40 ms and 3-view p50 at most 40 ms. Final 7D action +cosine must be at least 0.999 globally and 0.995 for every sample; internal raw +cosine must be at least 0.995 globally and for every sample. ## Encoder Down v7 Multi-View Result (commit `8424808`, 2026-07-26) diff --git a/examples/quickstart.py b/examples/quickstart.py index 5bb21c4d..2d437bc0 100644 --- a/examples/quickstart.py +++ b/examples/quickstart.py @@ -50,7 +50,8 @@ def main(): parser.add_argument('--benchmark', type=int, default=0) parser.add_argument('--warmup', type=int, default=500, help="Warmup iters before timed run. RTX 5090 needs " - "~500 to reach boost P-state; Thor can use 20.") + "~500 to reach boost P-state; Pi0.5 on Thor " + "may need ~300 for stable results.") parser.add_argument('--autotune', type=int, default=3, help="Autotune trials: 0=off, 3=default, 5=thorough") parser.add_argument('--recalibrate', action='store_true', @@ -130,7 +131,8 @@ def main(): if args.benchmark > 0: # RTX 5090 takes ~500 replay iterations to climb from idle P8 # (195 MHz) to boost P1 (~2870 MHz). Small warmup biases P50 by - # 2-3 ms. Jetson Thor settles faster, 50 is plenty there. + # 2-3 ms. Pi0.5 on Thor can also need ~300 iterations before the + # latency regime settles. warmup = args.warmup for _ in range(warmup): model.predict(images=imgs) diff --git a/tests/bench_pi05_decoder_fp4_e2e.py b/tests/bench_pi05_decoder_fp4_e2e.py index 8abee476..d2ef3419 100644 --- a/tests/bench_pi05_decoder_fp4_e2e.py +++ b/tests/bench_pi05_decoder_fp4_e2e.py @@ -26,8 +26,12 @@ EXPECTED_DEVICE = "NVIDIA Thor" EXPECTED_CC = (11, 0) -PUBLISHED_SOTA_P50_MS = {1: 30.5, 2: 36.3, 3: 42.8} -REQUIRED_MARGIN_MS = 2.0 +DEFAULT_WARMUP = 300 +LATENCY_GROUP_COUNT = 10 +RESULT_SCHEMA_VERSION = 2 +README_REFERENCE_P50_MS = {1: 23.01, 2: 27.17, 3: 31.74} +REGRESSION_BASELINE_P50_MS = {1: 30.5, 2: 36.3, 3: 42.8} +REQUIRED_REGRESSION_MARGIN_MS = 2.0 TAIL_LATENCY_MS = 40.0 PROMPT_TOKENS = [ 2, 18075, 908, 573, 3118, 3963, 578, @@ -60,6 +64,17 @@ } +def latency_group_medians(samples_ms: list[float]) -> list[float]: + """Keep timing groups in run order so late warmup remains visible.""" + if len(samples_ms) < LATENCY_GROUP_COUNT: + raise ValueError( + f"latency diagnostics require at least {LATENCY_GROUP_COUNT} samples") + return [ + float(np.median(group)) + for group in np.array_split(samples_ms, LATENCY_GROUP_COUNT) + ] + + def public_api_kwargs(mode: str, checkpoint: str, num_views: int) -> dict: """The exact load_model() call behind each published latency column.""" kwargs = dict( @@ -121,7 +136,10 @@ def main() -> int: "--fixture", help="exact N=8 fixture; defaults to the selected view count") parser.add_argument("--num-views", type=int, choices=(1, 2, 3), default=2) parser.add_argument("--output-dir") - parser.add_argument("--warmup", type=int, default=20) + parser.add_argument( + "--warmup", type=int, default=DEFAULT_WARMUP, + help="warmup calls per mode; approximately 300 are recommended for " + "stable Thor timing") parser.add_argument("--iters", type=int, default=100) parser.add_argument("--seed", type=int, default=20260725) parser.add_argument( @@ -400,6 +418,7 @@ def main() -> int: "p95_ms": float(np.percentile(latencies, 95)), "min_ms": min(latencies), "max_ms": max(latencies), + "latency_group_medians_ms": latency_group_medians(latencies), "samples_ms": latencies, } print("__E2E_RESULT__ " + json.dumps(result, sort_keys=True), flush=True) @@ -493,8 +512,9 @@ def main() -> int: fp8_p50 = float(child_results["fp8"]["p50_ms"]) fp4_p50 = float(child_results["fp4"]["p50_ms"]) fp4_p95 = float(child_results["fp4"]["p95_ms"]) - published_sota = PUBLISHED_SOTA_P50_MS[args.num_views] - target_latency = published_sota - REQUIRED_MARGIN_MS + readme_reference = README_REFERENCE_P50_MS[args.num_views] + regression_baseline = REGRESSION_BASELINE_P50_MS[args.num_views] + regression_limit = regression_baseline - REQUIRED_REGRESSION_MARGIN_MS comparison = { "raw_cosine": float( raw_lhs @ raw_rhs / @@ -512,9 +532,10 @@ def main() -> int: "p50_delta_ms": fp4_p50 - fp8_p50, "p50_speedup": fp8_p50 / fp4_p50, "num_views": args.num_views, - "published_sota_p50_ms": published_sota, - "required_margin_ms": REQUIRED_MARGIN_MS, - "target_p50_ms": target_latency, + "readme_reference_p50_ms": readme_reference, + "regression_baseline_p50_ms": regression_baseline, + "required_regression_margin_ms": REQUIRED_REGRESSION_MARGIN_MS, + "regression_limit_p50_ms": regression_limit, } gates = { "raw_cosine_at_least_0_995": comparison["raw_cosine"] >= 0.995, @@ -524,13 +545,14 @@ def main() -> int: "action_min_sample_cosine_at_least_0_995": ( comparison["action_min_sample_cosine"] >= 0.995), "fp4_p50_faster_than_fp8": fp4_p50 < fp8_p50, - "fp4_fa4_p50_beats_published_by_2ms": fp4_p50 <= target_latency, + "fp4_fa4_p50_within_regression_limit": fp4_p50 <= regression_limit, } if args.num_views == 2: gates["fp4_fa4_2v_p95_at_most_40_ms"] = fp4_p95 <= TAIL_LATENCY_MS if args.num_views == 3: gates["fp4_fa4_3v_p50_at_most_40_ms"] = fp4_p50 <= TAIL_LATENCY_MS result = { + "schema_version": RESULT_SCHEMA_VERSION, "timestamp_utc": datetime.now(timezone.utc).isoformat(), "commit": commit, "construction": args.construct, diff --git a/tests/test_pi05_thor_fp4_routing.py b/tests/test_pi05_thor_fp4_routing.py index 1a809936..3dea3aea 100644 --- a/tests/test_pi05_thor_fp4_routing.py +++ b/tests/test_pi05_thor_fp4_routing.py @@ -43,6 +43,30 @@ def _load_bench_module(): PRESET = BENCH.PUBLIC_API_PRESET +def test_benchmark_defaults_distinguish_reference_from_regression_gate(): + assert BENCH.DEFAULT_WARMUP == 300 + assert BENCH.RESULT_SCHEMA_VERSION == 2 + assert BENCH.README_REFERENCE_P50_MS == {1: 23.01, 2: 27.17, 3: 31.74} + assert BENCH.REGRESSION_BASELINE_P50_MS == {1: 30.5, 2: 36.3, 3: 42.8} + assert BENCH.REQUIRED_REGRESSION_MARGIN_MS == 2.0 + + +def test_benchmark_latency_groups_preserve_measurement_order(): + samples_ms = [ + value + for group in range(10) + for value in (float(group), float(group + 2)) + ] + assert BENCH.latency_group_medians(samples_ms) == [ + float(group + 1) for group in range(10) + ] + + +def test_benchmark_latency_groups_require_ten_samples(): + with pytest.raises(ValueError, match="at least 10 samples"): + BENCH.latency_group_medians([1.0] * 9) + + # The constructor parameters the Pi0.5 Thor frontends expose. load_model # feature-detects with inspect.signature, so the stubs must declare them for # the routing to be exercised at all. test_stub_signatures_match_frontends