diff --git a/.github/workflows/bench-verify.yml b/.github/workflows/bench-verify.yml index baf550bac..c4ca8fa15 100644 --- a/.github/workflows/bench-verify.yml +++ b/.github/workflows/bench-verify.yml @@ -26,10 +26,9 @@ jobs: startsWith(github.event.comment.body, '/bench-verify') && contains(fromJSON('["MEMBER","OWNER","COLLABORATOR"]'), github.event.comment.author_association) runs-on: [self-hosted, bench] - # Job cap. The verifier bench is ~5-6 min and the recursion measurement itself ~1 min, - # but on a cold runner the recursion BUILDS dominate: MEASURE_CLI (release cli) once, - # plus PER REF a guest build (~10-20 min) and a prover-test build for the blob dump. - # All cached in /tmp for later runs, and build-std / the host cargo target are shared + # Job cap. On a cold runner the recursion BUILDS dominate: MEASURE_CLI (release cli) + # once, plus PER REF a guest build and a prover-test build for the blob dump. All + # cached in /tmp for later runs, and build-std / the host cargo target are shared # across ref worktrees (see the recursion step's env) to keep the cold run under this # cap. The recursion step also has its own tighter timeout so a runaway build there # can't burn the whole job and lose the already-captured verifier result. @@ -46,7 +45,7 @@ jobs: await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: context.issue.number, - body: '⏳ **Benchmark started** on the bench server. The verifier bench takes ~5 min; the recursion-guest cycle comparison then adds guest builds — a few minutes when cached, up to ~1h on a cold run. The bench server is occupied until it finishes.' + body: '⏳ **Benchmark started** on the bench server. The recursion-guest cycle comparison adds guest builds on top of the verifier bench, longer on a cold runner. The bench server is occupied until it finishes.' }); - name: Resolve PR head + pair count @@ -92,10 +91,12 @@ jobs: scripts/bench_verify.sh "$HEAD_SHA" origin/main "$PAIRS" 2>&1 | tee /tmp/verify_out.txt sed -n '/=== Verify ABBA result/,$p' /tmp/verify_out.txt > /tmp/verify_result.txt - # Additive: deterministic recursion-guest cycle+accelerator diff (PR vs main). - # The measurement is one exact `execute --cycles` reading per ref (~1 min total, no - # ABBA). Cold wall time is dominated by BUILDS: MEASURE_CLI (release cli) once, plus - # per ref a guest build (~10-20 min) and a prover-test build for the blob dump; the + # Additive: deterministic recursion-guest cycle+accelerator diff (PR vs main), + # in three regimes: `min` (blowup=2, 1 query — cheap diagnostic floor) plus the + # realistic full-query base-layer points `blowup2` (219 queries) and `blowup4` + # (110 queries). Each measurement is one exact `execute --cycles` reading per ref + # (no ABBA). Cold wall time is dominated by BUILDS: MEASURE_CLI (release cli) + # once, plus a guest build and a prover-test build for the blob dump per ref; the # GUEST_TARGET_DIR / HOST_TARGET_DIR below share build-std and the host cargo target # across the two ref worktrees so the second ref is much cheaper (and per-worktree # disk shrinks). continue-on-error + `!cancelled()` keep this fully isolated from the @@ -105,11 +106,9 @@ jobs: id: recursion if: ${{ !cancelled() }} continue-on-error: true - # Fail-fast well under the 90-min job cap so a runaway recursion build can't burn + # Fail-fast well under the job cap so a runaway recursion build can't burn # the whole job and lose the already-captured verifier result (continue-on-error - # absorbs the timeout; the job still posts the verifier verdict). Sized with - # headroom over a cold shared-build run (MEASURE_CLI + 2 guest builds + 2 blob-dump - # builds). + # absorbs the timeout; the job still posts the verifier verdict). timeout-minutes: 70 env: HEAD_SHA: ${{ steps.cfg.outputs.head_sha }} @@ -122,6 +121,26 @@ jobs: set -o pipefail scripts/bench_recursion_cycles.sh "$HEAD_SHA" origin/main min 2>&1 | tee /tmp/recursion_out.txt sed -n '/=== Recursion-guest cycle/,$p' /tmp/recursion_out.txt > /tmp/recursion_result.txt + # Full-query regimes: the realistic base-layer proofs (the single-query `min` + # regime above is a diagnostic floor, not the real verifier cost). Guest builds + # and worktrees are already cached from the min run, so each adds only a blob + # dump and one execute per ref. Both refs' dump tests must understand + # RECURSION_DUMP_PRESET to produce a comparable blob for a given preset — until + # `origin/main` merges the preset-aware dump test, it can only ever dump `min` + # options, so every blowup2/blowup4 attempt against it would fail the same way. + # Check that support once, up front, instead of attempting (and failing) each + # preset in turn. + if git grep -q RECURSION_DUMP_PRESET origin/main -- prover/src/tests/ 2>/dev/null; then + for preset in blowup2 blowup4; do + if scripts/bench_recursion_cycles.sh "$HEAD_SHA" origin/main "$preset" 2>&1 | tee "/tmp/recursion_out_${preset}.txt"; then + { echo; sed -n '/=== Recursion-guest cycle/,$p' "/tmp/recursion_out_${preset}.txt"; } >> /tmp/recursion_result.txt + else + { echo; echo "_(${preset} full-query regime unavailable for these refs — see the workflow log.)_"; } >> /tmp/recursion_result.txt + fi + done + else + { echo; echo "_(blowup2/blowup4 full-query regimes need \`origin/main\` to have the preset-aware dump test (RECURSION_DUMP_PRESET) — not merged yet, so only \`min\` is compared for this PR.)_"; } >> /tmp/recursion_result.txt + fi - name: Post result if: always() diff --git a/Makefile b/Makefile index 12ac291f5..783817b2b 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ compile-programs compile-recursion-elfs clean-asm clean-rust clean-bench clean-shared \ clean-recursion-elfs clean test test-asm \ test-rust test-ethrex test-executor test-syscalls test-flamegraph flamegraph-prover test-profile-recursion test-profile-recursion-single test-profile-recursion-multi \ +test-profile-recursion-block \ test-fast test-prover test-prover-all test-prover-debug test-disk-spill test-math-cuda test-cuda-integration test-cuda-fallback \ test-prover-cuda test-prover-comprehensive-cuda \ bench-math-cuda bench-prover bench-prover-cuda build check clippy fmt lint regen-ethrex-fixtures \ @@ -56,13 +57,23 @@ RECURSION_GUESTS := empty fibonacci RECURSION_ARTIFACTS := $(addprefix $(RECURSION_ARTIFACTS_DIR)/, $(addsuffix .elf, $(RECURSION_GUESTS))) # The recursion verifier itself (bench_vs/lambda/recursion) requires picking -# exactly one of its `min`/`blowup8` Cargo features at build time (fixes the -# inner ProofOptions — see main.rs). Each preset builds its own distinctly -# named [[bin]] (recursion--bench) to its own artifact, via the -# define/foreach/eval below rather than the generic %.elf pattern rule. The -# distinct bin names also make the two `cp`s race-free under `make -j`. -RECURSION_VERIFIER_PRESETS := min blowup8 -RECURSION_VERIFIER_ARTIFACTS := $(addprefix $(RECURSION_ARTIFACTS_DIR)/recursion-, $(addsuffix .elf, $(RECURSION_VERIFIER_PRESETS))) +# exactly one of its preset Cargo features at build time (fixes the inner +# ProofOptions — see main.rs). Each preset builds its own distinctly named +# [[bin]] (recursion--bench) to its own artifact, via the +# define/foreach/eval below rather than the generic %.elf pattern rule. +# `required-features` on each [[bin]] is a subset match, not an exact-set +# match, so e.g. `--features "continuation min"` (the cont-min build) also +# satisfies plain `recursion-min-bench`'s `required-features = ["min"]` and +# cargo builds it too — racing with a concurrent `make -j` job building +# `recursion-min.elf` (`--features min`) to that same shared-target-dir path. +# `--bin $(2)` in build_guest_elf pins each invocation to its one target bin. +RECURSION_VERIFIER_PRESETS := min blowup2 blowup4 blowup8 +# Continuation-verifying variants (the guest's `continuation` feature): verify a +# multi-epoch ContinuationProof bundle instead of a monolithic VmProof. Kept to +# the presets the recursion benchmarks actually measure. +RECURSION_CONT_PRESETS := min blowup2 blowup4 +RECURSION_VERIFIER_ARTIFACTS := $(addprefix $(RECURSION_ARTIFACTS_DIR)/recursion-, $(addsuffix .elf, $(RECURSION_VERIFIER_PRESETS))) \ + $(addprefix $(RECURSION_ARTIFACTS_DIR)/recursion-cont-, $(addsuffix .elf, $(RECURSION_CONT_PRESETS))) # Override with: make ... SYSROOT_DIR=$HOME/.lambda-vm-sysroot # to install the sysroot in a user-writable location and avoid sudo. @@ -191,6 +202,7 @@ cd $(1) && \ -Z build-std=core,alloc,std,compiler_builtins,panic_abort \ -Z build-std-features=compiler-builtins-mem \ -Z json-target-spec \ + --bin $(2) \ $(3) cp $(SHARED_TARGET_DIR)/riscv64im-lambda-vm-elf/release/$(2) $@ endef @@ -215,7 +227,7 @@ $(BENCH_ARTIFACTS_DIR)/%.elf: FORCE | prepare-sysroot $(BENCH_ARTIFACTS_DIR) $(RECURSION_ARTIFACTS_DIR)/%.elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_DIR) $(call build_guest_elf,$(RECURSION_GUESTS_DIR)/$*,$*-bench) -# The recursion verifier's `min`/`blowup8` presets: same crate dir, one +# The recursion verifier's presets (RECURSION_VERIFIER_PRESETS): same crate dir, one # differently named [[bin]] per preset (recursion--bench, gated on that # preset's Cargo feature) -> a differently named artifact. Generated per preset # from RECURSION_VERIFIER_PRESETS via define/foreach/eval rather than a pattern @@ -236,6 +248,14 @@ $(RECURSION_ARTIFACTS_DIR)/recursion-$(1).elf: FORCE | prepare-sysroot $(RECURSI endef $(foreach preset,$(RECURSION_VERIFIER_PRESETS),$(eval $(call recursion_verifier_rule,$(preset)))) +# Continuation variants: same crate, `continuation` feature on top of the preset +# feature -> recursion-cont--bench -> recursion-cont-.elf. +define recursion_cont_verifier_rule +$(RECURSION_ARTIFACTS_DIR)/recursion-cont-$(1).elf: FORCE | prepare-sysroot $(RECURSION_ARTIFACTS_DIR) + $$(call build_guest_elf,$$(RECURSION_GUESTS_DIR)/recursion,recursion-cont-$(1)-bench,--features "continuation $(1)") +endef +$(foreach preset,$(RECURSION_CONT_PRESETS),$(eval $(call recursion_cont_verifier_rule,$(preset)))) + clean-asm: -rm -rf $(ASM_ARTIFACTS_DIR) @@ -278,6 +298,10 @@ test-profile-recursion-single: compile-recursion-elfs test-profile-recursion-multi: compile-recursion-elfs cargo test --package lambda-vm-prover --lib test_recursion_profile_multiquery -- --ignored --nocapture +# Real-block profile (ethrex, blowup=4/4 transfers), via the `continuation` guest. +test-profile-recursion-block: compile-recursion-elfs $(RUST_ARTIFACTS_DIR)/ethrex.elf + cargo test --package lambda-vm-prover --lib --release test_recursion_profile_blowup4_block -- --ignored --nocapture + # Regenerate the committed ethrex block fixtures (see tooling/ethrex-fixtures). # Run after bumping the ethrex rev; README checksums are refreshed automatically. regen-ethrex-fixtures: diff --git a/bench_vs/lambda/recursion/Cargo.toml b/bench_vs/lambda/recursion/Cargo.toml index 473e3107c..f612949a8 100644 --- a/bench_vs/lambda/recursion/Cargo.toml +++ b/bench_vs/lambda/recursion/Cargo.toml @@ -9,26 +9,57 @@ edition = "2024" # Exactly one selects the fixed ProofOptions (see main.rs) — hardcoded, not # private input, so a malicious input can't downgrade the security level. # Cargo features are additive by design, so the compile_error! mutual-exclusion -# guard in main.rs is the loud failure that stops a mislabeled artifact if both +# guard in main.rs is the loud failure that stops a mislabeled artifact if two # ever get enabled at once (e.g. under `--all-features`). The crate must stay a # standalone `[workspace]` (not a root-workspace member) so root-level feature -# unification can never turn both on. +# unification can never turn two on. min = [] +blowup2 = [] +blowup4 = [] blowup8 = [] +# Orthogonal to the presets: verify a ContinuationProof bundle (multi-epoch, +# memory-bounded inner prove) instead of a monolithic VmProof. Selects the +# `recursion-cont--bench` bins below. +continuation = [] # One distinctly named binary per preset (selected by its feature) so a parallel # `make -j` builds them to different filenames — structurally race-free, no cp -# clobbering. Both use src/main.rs; required-features gates each to its preset. +# clobbering. All use src/main.rs; required-features gates each to its preset. [[bin]] name = "recursion-min-bench" path = "src/main.rs" required-features = ["min"] +[[bin]] +name = "recursion-blowup2-bench" +path = "src/main.rs" +required-features = ["blowup2"] + +[[bin]] +name = "recursion-blowup4-bench" +path = "src/main.rs" +required-features = ["blowup4"] + [[bin]] name = "recursion-blowup8-bench" path = "src/main.rs" required-features = ["blowup8"] +[[bin]] +name = "recursion-cont-min-bench" +path = "src/main.rs" +required-features = ["continuation", "min"] + +[[bin]] +name = "recursion-cont-blowup2-bench" +path = "src/main.rs" +required-features = ["continuation", "blowup2"] + +[[bin]] +name = "recursion-cont-blowup4-bench" +path = "src/main.rs" +required-features = ["continuation", "blowup4"] + [dependencies] lambda-vm-prover = { path = "../../../prover", default-features = false, features = [ "profile-markers", diff --git a/bench_vs/lambda/recursion/src/main.rs b/bench_vs/lambda/recursion/src/main.rs index 33a061364..116ba54fd 100644 --- a/bench_vs/lambda/recursion/src/main.rs +++ b/bench_vs/lambda/recursion/src/main.rs @@ -12,15 +12,23 @@ //! `recursion::verify_and_attest_blob` — no deserialization pass, no owned //! `VmProof`. //! -//! `ProofOptions` is fixed by the `min`/`blowup8` Cargo feature (a `Preset`), -//! not private input — an attacker could otherwise pick trivially weak options -//! and have the guest accept as if a real proof had been checked. +//! The `continuation` feature swaps the monolithic proof for a multi-epoch +//! `ContinuationProof` bundle on the same wire format +//! (`recursion::ContinuationGuestInput`, built by +//! `recursion::encode_continuation_guest_input`), verified via +//! `recursion::verify_continuation_and_attest` — same trust model; the +//! bundle is materialized with one rkyv deserialize pass (zero-copy epoch +//! verify is follow-up work). //! -//! On success commits `program_id || inner_public_output` via -//! `recursion::verify_and_attest_blob` (a single ELF parse and a single -//! full-ELF Keccak, shared between the statement absorb and the `program_id` -//! fold). The id fold is what the consumer rebinds to a trusted ELF -//! (`check_attestation`); it is not self-enforcing here — the binding is +//! `ProofOptions` is fixed by exactly one preset Cargo feature +//! (`min`/`blowup2`/`blowup4`/`blowup8` — a `Preset`), not private input — an +//! attacker could otherwise pick trivially weak options and have the guest +//! accept as if a real proof had been checked. +//! +//! On success commits `program_id || inner_public_output` (a single ELF parse +//! and a single full-ELF Keccak, shared between the statement absorb and the +//! `program_id` fold). The id fold is what the consumer rebinds to a trusted +//! ELF (`check_attestation`); it is not self-enforcing here — the binding is //! established by the consumer via `recursion::check_attestation` (a //! host-side recompute+compare), never in-guest. //! @@ -31,14 +39,30 @@ use lambda_vm_prover::recursion::Preset; -#[cfg(not(any(feature = "min", feature = "blowup8")))] -compile_error!("select exactly one of the `min`/`blowup8` features"); -#[cfg(all(feature = "min", feature = "blowup8"))] -compile_error!("select exactly one of the `min`/`blowup8` features"); +#[cfg(not(any( + feature = "min", + feature = "blowup2", + feature = "blowup4", + feature = "blowup8" +)))] +compile_error!("select exactly one of the `min`/`blowup2`/`blowup4`/`blowup8` features"); +#[cfg(any( + all(feature = "min", feature = "blowup2"), + all(feature = "min", feature = "blowup4"), + all(feature = "min", feature = "blowup8"), + all(feature = "blowup2", feature = "blowup4"), + all(feature = "blowup2", feature = "blowup8"), + all(feature = "blowup4", feature = "blowup8"), +))] +compile_error!("select exactly one of the `min`/`blowup2`/`blowup4`/`blowup8` features"); /// The build preset fixing the inner `ProofOptions` (see the module docs). #[cfg(feature = "min")] const PRESET: Preset = Preset::Min; +#[cfg(feature = "blowup2")] +const PRESET: Preset = Preset::Blowup2; +#[cfg(feature = "blowup4")] +const PRESET: Preset = Preset::Blowup4; #[cfg(feature = "blowup8")] const PRESET: Preset = Preset::Blowup8; @@ -65,9 +89,17 @@ pub fn main() -> ! { // is what the consumer rebinds to a trusted ELF (`check_attestation`); it is // not self-enforcing here. let options = PRESET.options(); + + #[cfg(not(feature = "continuation"))] let attestation = lambda_vm_prover::recursion::verify_and_attest_blob(blob, &options) .expect("verify errored") .expect("inner proof failed verification"); + + #[cfg(feature = "continuation")] + let attestation = lambda_vm_prover::recursion::verify_continuation_and_attest(blob, &options) + .expect("verify errored") + .expect("inner continuation proof failed verification"); + lambda_vm_syscalls::syscalls::commit(&attestation); lambda_vm_syscalls::syscalls::sys_halt(); } diff --git a/executor/.gitignore b/executor/.gitignore index fa48867ab..ee277c65b 100644 --- a/executor/.gitignore +++ b/executor/.gitignore @@ -2,3 +2,9 @@ /program_artifacts/rust /tests/ethrex_hoodi.bin /tests/ethrex_bench_*.bin +# _4 is committed: prover/src/tests/recursion_smoke_test.rs reads it directly +# (~17 KB), and scripts/bench_recursion_scaling.sh also reads it as part of its +# scaling ladder. The other ladder sizes (_1/_8/_16) and bigger ones (e.g. _20) +# stay ignored — bench_recursion_scaling.sh generates any missing fixture on +# demand via tooling/ethrex-fixtures. +!/tests/ethrex_bench_4.bin diff --git a/executor/tests/ethrex_bench_4.bin b/executor/tests/ethrex_bench_4.bin new file mode 100644 index 000000000..45fe93038 Binary files /dev/null and b/executor/tests/ethrex_bench_4.bin differ diff --git a/prover/src/recursion.rs b/prover/src/recursion.rs index 654b58fa4..4bd390655 100644 --- a/prover/src/recursion.rs +++ b/prover/src/recursion.rs @@ -20,9 +20,10 @@ //! //! [`program_id`] deliberately does not fold the `ProofOptions`: the security //! level is pinned by which verifier guest the outer proof is checked against -//! (`recursion-min.elf` vs `recursion-blowup8.elf`, fixed at build time — see -//! [`Preset`]). A consumer must pin that outer ELF too, or a 1-query `min` -//! attestation is indistinguishable from a 128-bit `blowup8` one. +//! (`recursion-min.elf` vs `recursion-blowup2.elf`/`recursion-blowup4.elf`/ +//! `recursion-blowup8.elf`, fixed at build time — see [`Preset`]). A consumer +//! must pin that outer ELF too, or a 1-query `min` attestation is +//! indistinguishable from a 128-bit one. use crypto::hash::platform_keccak::PlatformKeccak256 as Keccak256; use digest::Digest; @@ -52,15 +53,37 @@ pub const MIN_PROOF_OPTIONS: ProofOptions = ProofOptions { pub enum Preset { /// Blowup=2, 1 query ([`MIN_PROOF_OPTIONS`]) — insecure, diagnostics only. Min, - /// Blowup=8, multi-query — 128-bit security. + /// Blowup=2, full 128-bit query count (219 queries at 20 grinding bits) — + /// the realistic base-layer shape: production pipelines prove the base + /// proof at low blowup (2/4) and reserve high blowup for the final wrap. + Blowup2, + /// Blowup=4, 110 queries — the other realistic base-layer point (e.g. + /// Zisk's compressor layer): 2× the prover LDE of blowup=2 for half the + /// queries to verify. + Blowup4, + /// Blowup=8, multi-query (73 queries) — 128-bit security at final-wrap-style + /// parameters: more prover work per row, far fewer queries to verify. Blowup8, } impl Preset { + /// Every preset, for name→preset lookups (e.g. the blob-dump test's + /// `RECURSION_DUMP_PRESET`). Keep in sync with the enum. + pub const ALL: [Preset; 4] = [ + Preset::Min, + Preset::Blowup2, + Preset::Blowup4, + Preset::Blowup8, + ]; + /// The fixed `ProofOptions` this preset's guest verifies with. pub fn options(&self) -> ProofOptions { match self { Preset::Min => MIN_PROOF_OPTIONS, + Preset::Blowup2 => crate::GoldilocksCubicProofOptions::with_blowup(2) + .expect("blowup=2 is always valid"), + Preset::Blowup4 => crate::GoldilocksCubicProofOptions::with_blowup(4) + .expect("blowup=4 is always valid"), Preset::Blowup8 => crate::GoldilocksCubicProofOptions::with_blowup(8) .expect("blowup=8 is always valid"), } @@ -71,6 +94,8 @@ impl Preset { pub fn artifact_stem(&self) -> &'static str { match self { Preset::Min => "recursion-min", + Preset::Blowup2 => "recursion-blowup2", + Preset::Blowup4 => "recursion-blowup4", Preset::Blowup8 => "recursion-blowup8", } } @@ -79,6 +104,8 @@ impl Preset { pub fn name(&self) -> &'static str { match self { Preset::Min => "min", + Preset::Blowup2 => "blowup2", + Preset::Blowup4 => "blowup4", Preset::Blowup8 => "blowup8", } } @@ -127,6 +154,49 @@ pub fn encode_guest_input( }) } +/// The continuation guest's private-input layout (the `continuation` guest +/// feature). Mirrors [`crate::GuestInput`] with the monolithic proof replaced +/// by the bundle and the PAGE roots replaced by the global-memory genesis +/// roots (see [`crate::continuation::continuation_precomputed_commitments`]). +/// Rkyv-archived on the same magic-prefixed wire format as the monolithic +/// blob ([`crate::encode_recursion_input`]); the guest is feature-pinned to +/// one layout, and a blob of the other kind fails the bytecheck validation. +#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] +pub struct ContinuationGuestInput { + pub bundle: crate::continuation::ContinuationProof, + pub inner_elf: Vec, + pub decode_commitment: Commitment, + pub page_commitments: Vec<(u64, Commitment)>, +} + +/// Build the continuation guest's private-input blob for `bundle` of +/// `inner_elf`: precomputes the roots and rkyv-encodes a +/// [`ContinuationGuestInput`] behind the standard aligning prefix. Takes the +/// bundle by value (it is large; the encoder is its last consumer). +pub fn encode_continuation_guest_input( + bundle: crate::continuation::ContinuationProof, + inner_elf: &[u8], + opts: &ProofOptions, +) -> Result, Error> { + let (decode_commitment, page_commitments) = + crate::continuation::continuation_precomputed_commitments(inner_elf, &bundle, opts)?; + let input = ContinuationGuestInput { + bundle, + inner_elf: inner_elf.to_vec(), + decode_commitment, + page_commitments, + }; + let archive = rkyv::to_bytes::(&input) + .map_err(|e| Error::Execution(format!("rkyv encode failed: {e}")))?; + let mut blob = Vec::with_capacity(crate::RECURSION_INPUT_PREFIX_LEN + archive.len()); + blob.extend_from_slice(&crate::RECURSION_INPUT_MAGIC); + blob.extend_from_slice(&crate::RECURSION_INPUT_VERSION.to_le_bytes()); + blob.extend_from_slice(&[0u8; 4]); // reserved + debug_assert_eq!(blob.len(), crate::RECURSION_INPUT_PREFIX_LEN); + blob.extend_from_slice(&archive); + Ok(blob) +} + /// Domain tag for [`program_id`]. const PROGRAM_ID_TAG: &[u8] = b"LAMBDAVM_PROGRAM_ID_V1"; @@ -223,49 +293,6 @@ pub fn verify_and_attest_blob( Ok(Some(attestation)) } -/// The continuation guest's private-input layout (the `continuation` guest -/// feature). Mirrors [`crate::GuestInput`] with the monolithic proof replaced -/// by the bundle and the PAGE roots replaced by the global-memory genesis -/// roots (see [`crate::continuation::continuation_precomputed_commitments`]). -/// Rkyv-archived on the same magic-prefixed wire format as the monolithic -/// blob ([`crate::encode_recursion_input`]); the guest is feature-pinned to -/// one layout, and a blob of the other kind fails the bytecheck validation. -#[derive(rkyv::Archive, rkyv::Serialize, rkyv::Deserialize)] -pub struct ContinuationGuestInput { - pub bundle: crate::continuation::ContinuationProof, - pub inner_elf: Vec, - pub decode_commitment: Commitment, - pub page_commitments: Vec<(u64, Commitment)>, -} - -/// Build the continuation guest's private-input blob for `bundle` of -/// `inner_elf`: precomputes the roots and rkyv-encodes a -/// [`ContinuationGuestInput`] behind the standard aligning prefix. Takes the -/// bundle by value (it is large; the encoder is its last consumer). -pub fn encode_continuation_guest_input( - bundle: crate::continuation::ContinuationProof, - inner_elf: &[u8], - opts: &ProofOptions, -) -> Result, Error> { - let (decode_commitment, page_commitments) = - crate::continuation::continuation_precomputed_commitments(inner_elf, &bundle, opts)?; - let input = ContinuationGuestInput { - bundle, - inner_elf: inner_elf.to_vec(), - decode_commitment, - page_commitments, - }; - let archive = rkyv::to_bytes::(&input) - .map_err(|e| Error::Execution(format!("rkyv encode failed: {e}")))?; - let mut blob = Vec::with_capacity(crate::RECURSION_INPUT_PREFIX_LEN + archive.len()); - blob.extend_from_slice(&crate::RECURSION_INPUT_MAGIC); - blob.extend_from_slice(&crate::RECURSION_INPUT_VERSION.to_le_bytes()); - blob.extend_from_slice(&[0u8; 4]); // reserved - debug_assert_eq!(blob.len(), crate::RECURSION_INPUT_PREFIX_LEN); - blob.extend_from_slice(&archive); - Ok(blob) -} - /// [`verify_and_attest_blob`]'s logic for a continuation bundle: takes the /// wire-format blob ([`encode_continuation_guest_input`]) and does the /// intended `continuation` guest's whole job in one call — verify every diff --git a/prover/src/tests/recursion_smoke_test.rs b/prover/src/tests/recursion_smoke_test.rs index bd1244c30..1fe7b9c91 100644 --- a/prover/src/tests/recursion_smoke_test.rs +++ b/prover/src/tests/recursion_smoke_test.rs @@ -155,9 +155,22 @@ fn drive_executor( (total_cycles, start.elapsed()) } +/// The identity + output a correct in-VM run must commit — the profile +/// tests' correctness oracle. Computed host-side (trustless recompute) before +/// the guest runs, then checked against the guest's actual committed +/// attestation once profiling finishes ([`run_profile_from`]). Mirrors the +/// production consumer check ([`recursion::check_attestation`]) for the +/// monolithic path, and its continuation analog +/// ([`crate::continuation::continuation_precomputed_commitments`] + +/// [`recursion::program_id_from_elf`]) for the continuation path. +struct ExpectedAttestation { + id: [u8; 32], + output: Vec, +} + /// Shared preamble: build the blob (an `empty` inner proof under the preset's /// options), load the `recursion-.elf` verifier, and stand up an -/// executor. Returns `(elf_bytes, program, executor)`. +/// executor. Returns `(elf_bytes, program, executor, expected_attestation)`. fn setup_guest_run( label: &str, preset: Preset, @@ -165,14 +178,21 @@ fn setup_guest_run( Vec, executor::elf::Elf, executor::vm::execution::Executor, + ExpectedAttestation, ) { let root = workspace_root(); let empty_elf_bytes = read_guest_elf(&root, "empty"); let guest_elf_bytes = read_guest_elf(&root, preset.artifact_stem()); - let (_inner_proof, blob) = + let (inner_proof, blob) = prove_inner_and_encode_blob(label, &empty_elf_bytes, &[], &preset.options()); + let expected = ExpectedAttestation { + id: recursion::expected_program_id(&empty_elf_bytes, &preset.options()) + .expect("expected_program_id errored"), + output: inner_proof.public_output, + }; + let program = executor::elf::Elf::load(&guest_elf_bytes).expect("ELF load failed"); assert_ne!( program.entry_point, @@ -182,7 +202,89 @@ fn setup_guest_run( ); let executor = executor::vm::execution::Executor::new(&program, blob).expect("Executor::new failed"); - (guest_elf_bytes, program, executor) + (guest_elf_bytes, program, executor, expected) +} + +/// Read the ethrex guest ELF (built by `make executor/program_artifacts/rust/ethrex.elf`). +fn read_ethrex_elf(root: &std::path::Path) -> Vec { + let path = root.join("executor/program_artifacts/rust/ethrex.elf"); + std::fs::read(&path).unwrap_or_else(|e| { + panic!( + "failed to read {} — run `make executor/program_artifacts/rust/ethrex.elf`: {e}", + path.display() + ) + }) +} + +/// Read a committed ethrex block fixture (`executor/tests/ethrex_bench_.bin`). +fn read_ethrex_fixture(root: &std::path::Path, txs: u32) -> Vec { + let path = root.join(format!("executor/tests/ethrex_bench_{txs}.bin")); + std::fs::read(&path).unwrap_or_else(|e| panic!("failed to read {}: {e}", path.display())) +} + +/// [`setup_guest_run`] for the `continuation` guest feature: proves `inner_elf` +/// on `inner_input` via continuations (`epoch_log2`-cycle epochs) and loads +/// `recursion-cont-.elf` instead of the monolithic `recursion-.elf` +/// — the realistic in-VM cost of verifying a real (multi-epoch) inner proof +/// instead of the `empty`-program diagnostic floor. Returns +/// `(elf_bytes, program, executor, expected_attestation)`. +fn setup_continuation_guest_run( + label: &str, + preset: Preset, + inner_elf_bytes: &[u8], + inner_input: &[u8], + epoch_log2: u32, +) -> ( + Vec, + executor::elf::Elf, + executor::vm::execution::Executor, + ExpectedAttestation, +) { + let root = workspace_root(); + let guest_elf_bytes = read_guest_elf(&root, &format!("recursion-cont-{}", preset.name())); + + let opts = preset.options(); + eprintln!( + "[{label}] proving inner continuation (blowup={}, fri_queries={}, epoch=2^{epoch_log2}) ...", + opts.blowup_factor, opts.fri_number_of_queries + ); + let bundle = + crate::continuation::prove_continuation(inner_elf_bytes, inner_input, epoch_log2, &opts) + .expect("inner continuation prove should succeed"); + eprintln!("[{label}] continuation epochs: {}", bundle.num_epochs()); + + // Ground truth, computed on the bundle before `encode_continuation_guest_input` + // consumes it: a full trustless host verify (mirrors the monolithic pipeline's + // host `verify_with_options` check) for the expected output, plus the + // continuation analog of `check_attestation`'s recompute for the expected id. + let expected_output = crate::continuation::verify_continuation(inner_elf_bytes, &bundle, &opts) + .expect("verify_continuation errored") + .expect("continuation bundle must verify on host before profiling the guest"); + let (expected_decode, expected_pages) = + crate::continuation::continuation_precomputed_commitments(inner_elf_bytes, &bundle, &opts) + .expect("continuation_precomputed_commitments errored"); + let expected_id = + recursion::program_id_from_elf(inner_elf_bytes, &expected_decode, &expected_pages) + .expect("program_id_from_elf errored"); + let expected = ExpectedAttestation { + id: expected_id, + output: expected_output, + }; + + let blob = recursion::encode_continuation_guest_input(bundle, inner_elf_bytes, &opts) + .expect("recursion::encode_continuation_guest_input failed"); + eprintln!("[{label}] rkyv blob: {} bytes", blob.len()); + + let program = executor::elf::Elf::load(&guest_elf_bytes).expect("ELF load failed"); + assert_ne!( + program.entry_point, + 0, + "recursion-cont-{} ELF has entry_point=0 — build artifact is malformed", + preset.name() + ); + let executor = + executor::vm::execution::Executor::new(&program, blob).expect("Executor::new failed"); + (guest_elf_bytes, program, executor, expected) } /// Demangled enclosing-function name for a PC via the ELF symbol table; @@ -324,16 +426,71 @@ fn print_step_breakdown(buckets: &[u64; 7], total_cycles: u64) { } } -/// Single-pass execute-only profiler. Always prints total cycles, the -/// per-step cycle breakdown (marker decode is cheap — one `InstructionCache` -/// lookup per cycle), and a rough trace/LDE estimate; with `detailed`, also -/// the top-25 functions table (needs a `pc_hist` HashMap, so gated). +/// Single-pass execute-only profiler over the `empty` inner program (the +/// verifier's intrinsic recursion overhead, not a real workload). Always +/// prints total cycles, the per-step cycle breakdown (marker decode is cheap — +/// one `InstructionCache` lookup per cycle), and a rough trace/LDE estimate; +/// with `detailed`, also the top-25 functions table (needs a `pc_hist` +/// HashMap, so gated). fn run_profile(preset: Preset, progress_stride: usize, detailed: bool) { + let (guest_elf_bytes, program, executor, expected) = setup_guest_run("profile", preset); + run_profile_from( + preset, + &guest_elf_bytes, + &program, + executor, + progress_stride, + detailed, + &expected, + ); +} + +/// [`run_profile`] over a REAL inner program instead of `empty`: verifies +/// `inner_elf_bytes`/`inner_input` via the `continuation` guest +/// (`recursion-cont-.elf`) — the realistic in-VM verifier cost, not +/// the intrinsic-overhead floor. +fn run_profile_continuation( + preset: Preset, + inner_elf_bytes: &[u8], + inner_input: &[u8], + epoch_log2: u32, + progress_stride: usize, + detailed: bool, +) { + let (guest_elf_bytes, program, executor, expected) = setup_continuation_guest_run( + "profile-block", + preset, + inner_elf_bytes, + inner_input, + epoch_log2, + ); + run_profile_from( + preset, + &guest_elf_bytes, + &program, + executor, + progress_stride, + detailed, + &expected, + ); +} + +/// Shared profiling loop for [`run_profile`]/[`run_profile_continuation`]: runs +/// an already-set-up guest executor and prints the same cycle/step/function +/// breakdown regardless of which inner program it verified. +fn run_profile_from( + preset: Preset, + guest_elf_bytes: &[u8], + program: &executor::elf::Elf, + mut executor: executor::vm::execution::Executor, + progress_stride: usize, + detailed: bool, + expected: &ExpectedAttestation, +) { use std::collections::HashMap; let opts = preset.options(); - let (guest_elf_bytes, program, mut executor) = setup_guest_run("profile", preset); - let symbols = executor::elf::SymbolTable::parse(&guest_elf_bytes); + let symbols = executor::elf::SymbolTable::parse(guest_elf_bytes); let instructions = executor::vm::execution::InstructionCache::new(&program.data) .expect("instruction cache build failed"); @@ -386,6 +543,32 @@ fn run_profile(preset: Preset, progress_stride: usize, detailed: bool) { }, ); + // Correctness, not just crash-freedom: read the guest's actual committed + // attestation and check it against the trusted host recompute + // (`expected`, built by `setup_guest_run`/`setup_continuation_guest_run` + // before the guest ran) — the same identity+output binding a production + // consumer checks via `check_attestation`/its continuation analog. + let committed = executor + .finish() + .expect("read committed output after execution") + .memory_values; + let (id, output) = recursion::split_attestation(&committed) + .expect("attestation too short (guest committed fewer than 32 bytes)"); + assert_eq!( + id, expected.id, + "guest attestation program_id mismatch — in-VM verify accepted a different \ + (ELF, roots) identity than the trusted host recompute" + ); + assert_eq!( + output, + expected.output.as_slice(), + "attested inner public output mismatch — the in-VM verify's committed output \ + diverges from the trusted host recompute" + ); + eprintln!( + "[profile] guest attestation matched the trusted host recompute (program_id + inner public output) ✓" + ); + eprintln!(); eprintln!("============================================================"); eprintln!( @@ -674,7 +857,7 @@ fn test_recursion_execute_1query() { #[test] #[ignore = "slow: runs the in-VM STARK verifier (minutes on CI)"] fn test_recursion_step_markers_observed_in_order() { - let (_bytes, program, mut executor) = setup_guest_run("step-markers", Preset::Min); + let (_bytes, program, mut executor, _expected) = setup_guest_run("step-markers", Preset::Min); let instructions = executor::vm::execution::InstructionCache::new(&program.data) .expect("instruction cache build failed"); @@ -771,19 +954,101 @@ fn test_recursion_prove_1query() { } /// Dump the guest's private-input blob to `/tmp/recursion_input.bin` for the -/// CLI's `execute --flamegraph`. +/// CLI's `execute --flamegraph` and `scripts/bench_recursion_cycles.sh`. +/// +/// Env knobs: +/// * `RECURSION_DUMP_PRESET` (`min`|`blowup2`|`blowup4`|`blowup8`, default +/// `min`) — the [`Preset`] whose options the inner proof is generated under; +/// must match the `recursion-.elf` the blob will be fed to, or the +/// guest rejects the proof. +/// * `RECURSION_DUMP_INNER_ELF` (path, default the `empty` guest) — the inner +/// program to prove, for realistic trace heights (e.g. the ethrex guest). +/// * `RECURSION_DUMP_INNER_INPUT` (path, default none) — the inner program's +/// private input (e.g. an ethrex-fixtures block). +/// * `RECURSION_DUMP_EPOCH_LOG2` (int, default unset = monolithic) — prove the +/// inner via continuations with `2^n`-cycle epochs (memory-bounded prover) +/// and encode a [`recursion::ContinuationGuestInput`] blob instead; feed it +/// to `recursion-cont-.elf`, not the monolithic guest. #[test] #[ignore = "diagnostic: writes recursion private input to /tmp/recursion_input.bin"] fn test_dump_recursion_input() { let root = workspace_root(); - let empty_elf_bytes = read_guest_elf(&root, "empty"); - let (_inner_proof, blob) = - prove_inner_and_encode_blob("dump-input", &empty_elf_bytes, &[], &MIN_PROOF_OPTIONS); + let preset_name = std::env::var("RECURSION_DUMP_PRESET").unwrap_or_else(|_| "min".to_string()); + let preset = Preset::ALL + .into_iter() + .find(|p| p.name() == preset_name) + .unwrap_or_else(|| { + panic!( + "unknown RECURSION_DUMP_PRESET '{preset_name}' (expected min|blowup2|blowup4|blowup8)" + ) + }); + + let (inner_elf_bytes, inner_label) = match std::env::var("RECURSION_DUMP_INNER_ELF") { + Ok(p) => ( + std::fs::read(&p).unwrap_or_else(|e| panic!("read RECURSION_DUMP_INNER_ELF {p}: {e}")), + p, + ), + Err(_) => (read_guest_elf(&root, "empty"), "empty".to_string()), + }; + let inner_input = match std::env::var("RECURSION_DUMP_INNER_INPUT") { + Ok(p) => { + std::fs::read(&p).unwrap_or_else(|e| panic!("read RECURSION_DUMP_INNER_INPUT {p}: {e}")) + } + Err(_) => Vec::new(), + }; + + let blob = match std::env::var("RECURSION_DUMP_EPOCH_LOG2") { + Ok(s) => { + // No recursion-cont-blowup8.elf is built (RECURSION_CONT_PRESETS stops + // at blowup4), so this blob would have no guest to verify it. + assert_ne!( + preset, + Preset::Blowup8, + "RECURSION_DUMP_PRESET=blowup8 has no recursion-cont-blowup8.elf guest; \ + continuation mode only supports min|blowup2|blowup4" + ); + let epoch_log2: u32 = s + .parse() + .unwrap_or_else(|e| panic!("bad RECURSION_DUMP_EPOCH_LOG2 '{s}': {e}")); + let opts = preset.options(); + eprintln!( + "[dump-input] proving inner continuation (blowup={}, fri_queries={}, epoch=2^{epoch_log2}) ...", + opts.blowup_factor, opts.fri_number_of_queries + ); + let bundle = crate::continuation::prove_continuation( + &inner_elf_bytes, + &inner_input, + epoch_log2, + &opts, + ) + .expect("inner continuation prove should succeed"); + eprintln!("[dump-input] continuation epochs: {}", bundle.num_epochs()); + recursion::encode_continuation_guest_input(bundle, &inner_elf_bytes, &opts) + .expect("recursion::encode_continuation_guest_input failed") + } + Err(_) => { + let (_inner_proof, blob) = prove_inner_and_encode_blob( + "dump-input", + &inner_elf_bytes, + &inner_input, + &preset.options(), + ); + blob + } + }; + assert!( + blob.len() <= executor::vm::memory::MAX_PRIVATE_INPUT_SIZE as usize, + "recursion input exceeds MAX_PRIVATE_INPUT_SIZE" + ); let path = "/tmp/recursion_input.bin"; std::fs::write(path, &blob).expect("write blob"); - eprintln!("[dump-input] wrote {} bytes to {path}", blob.len()); + eprintln!( + "[dump-input] preset={} inner={inner_label} wrote {} bytes to {path}", + preset.name(), + blob.len() + ); } /// Cycle count only of the recursion guest verifying a 1-query inner proof. @@ -800,6 +1065,49 @@ fn test_recursion_cycles_multiquery() { run_profile(Preset::Blowup8, 500, false); } +/// Cycle count only at 128-bit security with the realistic base-layer shape: +/// blowup=2 yields ~0.49 bits/query, so the full 219-query FRI dominates. +#[test] +#[ignore = "diagnostic: recursion guest cycle count (blowup=2, 219 queries)"] +fn test_recursion_cycles_blowup2() { + run_profile(Preset::Blowup2, 500, false); +} + +/// Cycle count only at 128-bit security, blowup=4 (110 queries) — the other +/// realistic base-layer point. +#[test] +#[ignore = "diagnostic: recursion guest cycle count (blowup=4, 110 queries)"] +fn test_recursion_cycles_blowup4() { + run_profile(Preset::Blowup4, 500, false); +} + +/// Continuation epoch size for the real-block profile below — matches +/// `scripts/bench_recursion_scaling.sh`'s default. +const BLOCK_EPOCH_LOG2: u32 = 21; + +/// Full profile (top-25 + per-step) of the recursion `continuation` guest +/// verifying a REAL ethrex block (4 transfers) — not the `empty`-program +/// diagnostic floor `test_recursion_profile_1query`/`_multiquery` measure. +/// blowup=4 (110 queries): the cheaper of the two realistic base-layer +/// presets, so the full histogram stays tractable; `bench_recursion_cycles.sh`/ +/// `bench_recursion_scaling.sh` already cover cycle counts across every preset +/// and block size, so this only needs to exist for the detailed breakdown. +#[test] +#[ignore = "diagnostic: heavy; recursion guest histogram + steps over a real ethrex block (blowup=4)"] +fn test_recursion_profile_blowup4_block() { + let root = workspace_root(); + let ethrex_elf_bytes = read_ethrex_elf(&root); + let fixture = read_ethrex_fixture(&root, 4); + run_profile_continuation( + Preset::Blowup4, + ðrex_elf_bytes, + &fixture, + BLOCK_EPOCH_LOG2, + 500, + true, + ); +} + /// Full profile (top-25 + per-step) of the 1-query run. #[test] #[ignore = "diagnostic: ~8 min; recursion guest histogram + steps (1 query)"] diff --git a/scripts/bench_recursion_cycles.sh b/scripts/bench_recursion_cycles.sh index 5db264a45..2235c9158 100755 --- a/scripts/bench_recursion_cycles.sh +++ b/scripts/bench_recursion_cycles.sh @@ -18,8 +18,8 @@ # single measuring CLI (MEASURE_CLI) built once from the checkout this script runs in: # * Guest cycles — retired instructions. # * Keccak calls — keccak-permutation accelerator ecalls (one cycle each, but each -# runs a whole permutation invisibly, so it's the companion signal; -# currently 0 until the verifier is wired to the keccak syscall). +# runs a whole permutation invisibly, so it's the companion signal: +# the verifier's Merkle/transcript hashing rides on this syscall). # The CLI also prints an Ecsm (EC scalar-mul) count, but the STARK verifier does no # scalar-mul, so it is structurally 0 for a recursion proof — dropped as noise, not read. # MEASURE_CLI's executor counts ANY ref's guest ELF correctly (it just feeds the blob @@ -35,14 +35,22 @@ # Usage: scripts/bench_recursion_cycles.sh REF_A [REF_B=origin/main] [PRESET=min] # REF_A ref/SHA to evaluate (the PR side). # REF_B baseline ref/SHA (default origin/main). -# PRESET recursion-verifier preset (default min). Per ref the tool prefers -# recursion-.elf and falls back to recursion.elf (older refs / main -# build a single unnamed recursion guest). It is EXPECTED and correct for the -# two sides to use DIFFERENT artifacts — e.g. main→recursion.elf while a -# preset PR→recursion-min.elf — because both are verified under the SAME min -# proof options (the dump test pins MIN_PROOF_OPTIONS). The printed per-ref -# `guest=` labels show which each side used; a differing name is the -# expected comparison, not a mismatch. +# PRESET recursion-verifier preset (default min): min = blowup=2, 1 query +# (cheap diagnostic regime); blowup2 = blowup=2, 219 queries (the +# realistic base-layer proof shape at 128-bit — production pipelines +# prove the base at low blowup and wrap at high blowup); blowup4 = +# blowup=4, 110 queries (the other realistic base-layer point); +# blowup8 = blowup=8, 73 queries. The preset picks BOTH the guest ELF +# (recursion-.elf, falling back to recursion.elf on older refs +# that build a single unnamed min guest) AND the inner-proof options of +# the dumped blob (exported as RECURSION_DUMP_PRESET to the dump test). +# Refs predating the preset-aware dump test always dump min options, so +# only PRESET=min is measurable for them — the script fails loudly +# up front rather than let the guest reject the blob in-VM. It is +# EXPECTED and correct for the two sides to use DIFFERENT artifact +# names (e.g. main→recursion.elf vs PR→recursion-min.elf) — both are +# verified under the SAME preset options. The printed per-ref +# `guest=` labels show which each side used. # Env: # REBUILD=1 force rebuild of MEASURE_CLI and re-run of every ref # (guest build + blob dump + measurement); ignore caches. @@ -104,8 +112,8 @@ prune_worktree_cache() { echo "==> Pruning old ref worktree $wt (keeping newest $PRUNE_KEEP)" >&2 git worktree remove --force "$wt" >/dev/null 2>&1 || rm -rf "$wt" rm -f "$WORK"/result_"${s8}"_*.txt "$WORK"/blob_"${s8}"_*.bin \ - "$WORK"/build_guest_"${s8}".log "$WORK"/dump_"${s8}".log \ - "$WORK"/measure_"${s8}".err + "$WORK"/build_guest_"${s8}".log "$WORK"/dump_"${s8}"*.log \ + "$WORK"/measure_"${s8}"*.err done <<< "$stale" git worktree prune >/dev/null 2>&1 || true } @@ -198,7 +206,7 @@ measure_ref() { # 2a. Build the recursion guest ELF(s) (+ empty.elf inner program). GUEST_TARGET_DIR, # when set, shares the RV64 build dir across ref worktrees (reuses build-std). - echo "==> [$role] make compile-recursion-elfs @ $sha8 (this can take 10-20 min the first time) ..." >&2 + echo "==> [$role] make compile-recursion-elfs @ $sha8 (slow the first time) ..." >&2 local glog="$WORK/build_guest_${sha8}.log" local -a make_args=(compile-recursion-elfs) if [ -n "${GUEST_TARGET_DIR:-}" ]; then @@ -229,22 +237,29 @@ measure_ref() { fi echo "==> [$role] guest ELF: $(basename "$guest_elf")" >&2 - # 2c. Generate this ref's own input blob via its ignored dump test. + # 2c. Generate this ref's own input blob via its ignored dump test, under this + # preset's options (RECURSION_DUMP_PRESET). Refs predating the preset-aware dump + # test always prove the min options; feeding that blob to a non-min guest would + # only fail in-VM verification much later, so refuse loudly up front instead. if ! grep -rq "fn test_dump_recursion_input" "$wt/prover/src/tests/" 2>/dev/null; then echo "ERROR: [$role] ref $ref ($sha8) has no 'test_dump_recursion_input' — cannot generate its input blob." >&2 exit 1 fi - echo "==> [$role] dumping recursion input blob (cargo test test_dump_recursion_input) ..." >&2 + if [ "$PRESET" != "min" ] && ! grep -rq "RECURSION_DUMP_PRESET" "$wt/prover/src/tests/" 2>/dev/null; then + echo "ERROR: [$role] ref $ref ($sha8) predates the preset-aware dump test (no RECURSION_DUMP_PRESET) — only PRESET=min is measurable for it." >&2 + exit 1 + fi + echo "==> [$role] dumping recursion input blob (cargo test test_dump_recursion_input, preset=$PRESET) ..." >&2 rm -f /tmp/recursion_input.bin - local dlog="$WORK/dump_${sha8}.log" + local dlog="$WORK/dump_${sha8}_${PRESET}.log" if [ -n "${HOST_TARGET_DIR:-}" ]; then - if ! ( cd "$wt" && CARGO_TARGET_DIR="$HOST_TARGET_DIR" cargo test -p lambda-vm-prover --lib test_dump_recursion_input -- --ignored --nocapture ) >"$dlog" 2>&1; then + if ! ( cd "$wt" && RECURSION_DUMP_PRESET="$PRESET" CARGO_TARGET_DIR="$HOST_TARGET_DIR" cargo test --release -p lambda-vm-prover --lib test_dump_recursion_input -- --ignored --nocapture ) >"$dlog" 2>&1; then echo "ERROR: [$role] blob-dump test failed for $ref ($sha8). Tail of $dlog:" >&2 tail -40 "$dlog" >&2 exit 1 fi else - if ! ( cd "$wt" && cargo test -p lambda-vm-prover --lib test_dump_recursion_input -- --ignored --nocapture ) >"$dlog" 2>&1; then + if ! ( cd "$wt" && RECURSION_DUMP_PRESET="$PRESET" cargo test --release -p lambda-vm-prover --lib test_dump_recursion_input -- --ignored --nocapture ) >"$dlog" 2>&1; then echo "ERROR: [$role] blob-dump test failed for $ref ($sha8). Tail of $dlog:" >&2 tail -40 "$dlog" >&2 exit 1 @@ -262,9 +277,9 @@ measure_ref() { echo "==> [$role] measuring: $MEASURE_CLI execute $(basename "$guest_elf") --private-input --cycles" >&2 local t0 t1 dt out t0=$(date +%s) - if ! out="$("$MEASURE_CLI" execute "$guest_elf" --private-input "$blob" --cycles 2>"$WORK/measure_${sha8}.err")"; then + if ! out="$("$MEASURE_CLI" execute "$guest_elf" --private-input "$blob" --cycles 2>"$WORK/measure_${sha8}_${PRESET}.err")"; then echo "ERROR: [$role] MEASURE_CLI execute failed for $ref ($sha8). Tail of stderr:" >&2 - tail -20 "$WORK/measure_${sha8}.err" >&2 + tail -20 "$WORK/measure_${sha8}_${PRESET}.err" >&2 exit 1 fi t1=$(date +%s); dt=$((t1 - t0)) @@ -334,10 +349,13 @@ mcycd() { } # Human label for the proof regime this preset measures, so a reader can't mistake the -# single-query `min` number for the full 128-bit verifier cost. CI always passes `min`. +# single-query `min` number for the full 128-bit verifier cost. CI passes `min` plus +# the full-query regimes `blowup2`/`blowup4` (see .github/workflows/bench-verify.yml). case "$PRESET" in min) REGIME="single query (blowup=2, 1 query)" ;; - blowup8) REGIME="128-bit (blowup=8, multi-query)" ;; + blowup2) REGIME="128-bit (blowup=2, 219 queries — realistic base-layer)" ;; + blowup4) REGIME="128-bit (blowup=4, 110 queries — realistic base-layer)" ;; + blowup8) REGIME="128-bit (blowup=8, 73 queries)" ;; *) REGIME="$PRESET" ;; esac diff --git a/scripts/bench_recursion_scaling.sh b/scripts/bench_recursion_scaling.sh new file mode 100755 index 000000000..11a3d67f5 --- /dev/null +++ b/scripts/bench_recursion_scaling.sh @@ -0,0 +1,121 @@ +#!/usr/bin/env bash +# +# bench_recursion_scaling.sh — in-VM recursion-verifier scaling ladder. +# +# Sweeps ethrex block sizes × verifier presets: proves each block's inner +# execution via CONTINUATIONS (memory-bounded 2^EPOCH_LOG2-cycle epochs, so any +# block size proves on a bounded-RAM box), then executes the continuation +# recursion guest (recursion-cont-.elf) on the bundle and records the +# EXACT deterministic guest cycle count — the in-VM cost of verifying that +# block's proof. +# +# Sweep order is PRESET-MAJOR on purpose: the full block-size curve for the +# first preset completes before the next preset starts, so the headline regime +# (blowup2 = the realistic base-layer options, 219 FRI queries) yields a usable +# scaling curve as early as possible instead of only when everything ends. +# +# Usage: scripts/bench_recursion_scaling.sh [RESULTS_FILE=/tmp/recursion_scaling.txt] +# Env: +# TXS="1 4 8 16" block sizes (transfers); fixtures are read from +# executor/tests/ethrex_bench_.bin (only _4 is +# committed) and generated via tooling/ethrex-fixtures +# when missing. +# PRESETS="blowup2 blowup4 min" verifier presets, most important first. +# EPOCH_LOG2=21 inner continuation epoch size (log2 cycles). +# DUMP_FEATURES="" extra cargo features for the proving dump, +# e.g. DUMP_FEATURES=cuda on a GPU box. +# +# Prereqs (the script fails fast on each): +# cargo build --release -p cli +# make compile-recursion-elfs (recursion-cont-*.elf) +# make executor/program_artifacts/rust/ethrex.elf (the inner guest) +# +# Output: one key=value line per cell in RESULTS_FILE, e.g. +# txs=4 preset=blowup2 epochs=2 blob=145080513 cycles=18984803380 keccak=3152604 exec_wall_s=164 +# Cycle counts are deterministic (machine-independent); wall times are not. +set -euo pipefail + +ROOT="$(git rev-parse --show-toplevel)" +cd "$ROOT" + +TXS="${TXS:-1 4 8 16}" +PRESETS="${PRESETS:-blowup2 blowup4 min}" +EPOCH_LOG2="${EPOCH_LOG2:-21}" +RESULTS="${1:-/tmp/recursion_scaling.txt}" +WORK="$(mktemp -d /tmp/recursion_scaling.XXXXXX)" +trap 'rm -rf "$WORK"' EXIT + +CLI=target/release/cli +ART=executor/program_artifacts/recursion +ETHREX=executor/program_artifacts/rust/ethrex.elf + +[ -x "$CLI" ] || { echo "ERROR: $CLI missing — run: cargo build --release -p cli" >&2; exit 1; } +[ -f "$ETHREX" ] || { echo "ERROR: $ETHREX missing — run: make $ETHREX" >&2; exit 1; } +for P in $PRESETS; do + [ -f "$ART/recursion-cont-${P}.elf" ] || { + echo "ERROR: $ART/recursion-cont-${P}.elf missing — run: make compile-recursion-elfs" >&2 + exit 1 + } +done + +echo "==> results -> $RESULTS (work dir: $WORK)" +: > "$RESULTS" + +for P in $PRESETS; do + for N in $TXS; do + FIX=executor/tests/ethrex_bench_${N}.bin + if [ ! -f "$FIX" ]; then + echo "==> [${P}/${N}tx] generating missing fixture $FIX" >&2 + ( cd tooling/ethrex-fixtures && cargo build --release ) >"$WORK/fixtures_build.log" 2>&1 + tooling/ethrex-fixtures/target/release/ethrex-fixtures "$N" "$FIX" distinct >&2 + fi + + # Inner block cost, once per block size (cheap; deterministic). + if ! ic="$("$CLI" execute "$ETHREX" --private-input "$FIX" --cycles | awk -F': ' '/^Cycles:/{print $2; exit}')" || [ -z "$ic" ]; then + echo "txs=$N preset=$P inner_cycles=FAILED" >> "$RESULTS" + echo "ERROR: [${P}/${N}tx] inner cycle measurement failed for $FIX" >&2 + continue + fi + + echo "==> [${P}/${N}tx] proving inner continuation (epoch=2^${EPOCH_LOG2}) ..." >&2 + rm -f /tmp/recursion_input.bin + DLOG="$WORK/dump_${N}tx_${P}.log" + if ! ( RECURSION_DUMP_PRESET="$P" RECURSION_DUMP_EPOCH_LOG2="$EPOCH_LOG2" \ + RECURSION_DUMP_INNER_ELF="$PWD/$ETHREX" RECURSION_DUMP_INNER_INPUT="$PWD/$FIX" \ + cargo test --release -p lambda-vm-prover ${DUMP_FEATURES:+--features "$DUMP_FEATURES"} --lib test_dump_recursion_input -- --ignored --nocapture ) \ + >"$DLOG" 2>&1 || [ ! -f /tmp/recursion_input.bin ]; then + echo "txs=$N preset=$P inner_cycles=$ic DUMP_FAILED" >> "$RESULTS" + echo "ERROR: [${P}/${N}tx] dump failed; tail of $DLOG:" >&2 + tail -20 "$DLOG" >&2 + continue + fi + epochs="$(grep -o 'continuation epochs: [0-9]*' "$DLOG" | awk '{print $3}')" + if [ -z "$epochs" ]; then + echo "txs=$N preset=$P inner_cycles=$ic EPOCHS_PARSE_FAILED" >> "$RESULTS" + echo "ERROR: [${P}/${N}tx] could not parse epoch count from $DLOG" >&2 + continue + fi + BLOB="$WORK/blob_${N}tx_${P}.bin" + mv /tmp/recursion_input.bin "$BLOB" + sz="$(wc -c < "$BLOB" | tr -d ' ')" + + echo "==> [${P}/${N}tx] executing recursion-cont-${P}.elf (${epochs} epochs, ${sz} bytes) ..." >&2 + t0=$(date +%s) + if out="$("$CLI" execute "$ART/recursion-cont-${P}.elf" --private-input "$BLOB" --cycles 2>"$WORK/exec_${N}tx_${P}.err")"; then + t1=$(date +%s) + cyc="$(printf '%s\n' "$out" | awk -F': ' '/^Cycles:/{print $2; exit}')" + kec="$(printf '%s\n' "$out" | awk -F': ' '/^Keccak calls:/{print $2; exit}')" + line="txs=$N preset=$P inner_cycles=$ic epochs=$epochs blob=$sz cycles=$cyc keccak=$kec exec_wall_s=$((t1 - t0))" + echo "$line" >> "$RESULTS" + echo " $line" >&2 + else + echo "txs=$N preset=$P inner_cycles=$ic epochs=$epochs blob=$sz EXEC_FAILED" >> "$RESULTS" + echo "ERROR: [${P}/${N}tx] guest execute failed; tail of stderr:" >&2 + tail -10 "$WORK/exec_${N}tx_${P}.err" >&2 + fi + rm -f "$BLOB" + done +done + +echo "==> done. Results:" +cat "$RESULTS"