diff --git a/Ix/Aiur/Protocol.lean b/Ix/Aiur/Protocol.lean index 705f2565..40d61c4f 100644 --- a/Ix/Aiur/Protocol.lean +++ b/Ix/Aiur/Protocol.lean @@ -104,6 +104,27 @@ opaque vkBytes : @& AiurSystem → ByteArray @[extern "rs_aiur_system_circuit_shapes"] opaque circuitShapes : @& AiurSystem → Array CircuitShape +/-- Scan-and-cut sharding against a Rust-owned `EnvHandle`, cutting on + the system's analytic peak-prove-RAM prediction (the system carries + the toplevel, so none is passed). Numeric params are decimal strings: + budget (GiB), eps (percent), workers (0 = autoscale), fail-fast ("0" + skips kernel-rejected blocks into a `.failed.csv`). Writes the `.ixes` + manifest and its costs sidecar to the output path. -/ +@[extern "rs_aiur_scan_shards_with_env"] +opaque scanShardsWithEnv : @& AiurSystem → + @& Bytecode.FunIdx → @& EnvHandle → @& String → @& String → @& String → + @& String → @& String → @& String → @& String → Except String Unit + +/-- The child side of the scan's process pool: run the stdin/stdout + worker loop until EOF (see `crates/ffi/src/aiur/scan.rs`, + `scan_worker`). Spawned by the parent scan as `ix shard-worker`; + numeric params are decimal strings: cut (GiB), batch blocks, soft + record cut (GiB), schedule pieces, exec-only ("1"/"0"). -/ +@[extern "rs_aiur_scan_worker"] +opaque scanWorker : @& AiurSystem → + @& Bytecode.FunIdx → @& EnvHandle → @& String → @& String → @& String → + @& String → @& String → Except String Unit + @[extern "rs_aiur_system_prove"] private opaque prove' : @& AiurSystem → @& Bytecode.FunIdx → @& Array G → diff --git a/Ix/Aiur/Semantics/BytecodeFfi.lean b/Ix/Aiur/Semantics/BytecodeFfi.lean index fcd920e7..22c8b8bf 100644 --- a/Ix/Aiur/Semantics/BytecodeFfi.lean +++ b/Ix/Aiur/Semantics/BytecodeFfi.lean @@ -211,6 +211,16 @@ def shardCheckWithEnv (toplevel : @& Bytecode.Toplevel) (shardCheckWithEnv' toplevel funIdx envHandle ownedBlob useBytecode).map fun r => (r.output, .ofArrays r.ioData r.ioMap, r.queryCounts) +/-- Execute-only whole-env check through the codegen'd Aiur kernel: no + partition, no manifest — the check verdict plus measured totals, + reported on stderr. Args: workers (0 = autoscale), fail-fast ("0" + records and skips kernel-rejected blocks; anything else aborts on + the first). -/ +@[extern "rs_aiur_execute_env_with_env"] +opaque executeEnvWithEnv : @& Bytecode.Toplevel → + @& Bytecode.FunIdx → @& EnvHandle → @& String → @& String → @& String → + @& String → Except String Unit + end Bytecode.Toplevel end Aiur diff --git a/Ix/Cli/AddrOfCmd.lean b/Ix/Cli/AddrOfCmd.lean index c6fb5076..839048a2 100644 --- a/Ix/Cli/AddrOfCmd.lean +++ b/Ix/Cli/AddrOfCmd.lean @@ -1,12 +1,15 @@ /- - `ix addr-of [--ixe ]`: resolve a Lean.Name to its - 32-byte content address. Without `--ixe`, the lookup compiles the - name's transitive closure from the compiled-in Lean env (via - `IxVM.ClaimHarness.loadIxonEnv` → `lookupAddr`). With `--ixe`, the + `ix addr-of [--ixe ] [--ixes ]`: resolve a + Lean.Name to its 32-byte content address. Without `--ixe`, the lookup + compiles the name's transitive closure from the compiled-in Lean env + (via `IxVM.ClaimHarness.loadIxonEnv` → `lookupAddr`). With `--ixe`, the lookup reads the env from disk and dispatches `Ixon.Env.getAddr?`. Prints the resulting address hex on stdout (one line, no prefix), so the output can be piped into `ix claim check $(ix addr-of …)` etc. + With `--ixes` (requires `--ixe`), a second line reports which shard of + the manifest's partition owns the name's check-schedule block — the + shard whose prove type-checks this constant. -/ module public import Cli @@ -16,6 +19,7 @@ public import Ix.Environment public import Ix.IxVM.ClaimHarness public import Ix.Ixon public import Ix.Meta +public import Ix.Cli.CheckCmd public import Ix.Cli.NameResolve public section @@ -42,7 +46,33 @@ def runAddrOfCmd (p : Cli.Parsed) : IO UInt32 := do | none => IO.eprintln s!"error: {name} not found in {path}"; return 1 | some addr => - IO.println (toString addr); return 0 + IO.println (toString addr) + if let some manifestPath := (p.flag? "ixes").map (·.as! String) then + -- Owning-shard lookup: the constant's check-schedule block (a + -- projection collapses to its SCC/Muts wrapper), searched in the + -- manifest's owned-block lists. + let c? : Option Ixon.Constant := Id.run do + for (a, lc) in ixonEnv.consts do + if a == addr then return lc.get? + return none + let some c := c? + | IO.eprintln s!"error: {addr} has no parseable constant in {path}" + return 1 + let block := Ix.Cli.CheckCmd.blockAddrOf addr c + match Ix.Cli.CheckCmd.parseIxesShards + (← IO.FS.readBinFile manifestPath) with + | .error e => + IO.eprintln s!"error: {manifestPath}: {e}"; return 1 + | .ok shards => + match (shards.mapIdx (fun k s => (s, k))).find? + (fun (s, _) => s.blocks.contains block) with + | some (s, k) => + IO.println s!"block {block} → shard {k} \ + ({s.blocks.size} blocks, cost {s.cost})" + | none => + IO.println s!"block {block} → no owning shard \ + (excluded from the partition)" + return 0 | none => let env ← get_env! if !env.constants.contains name then @@ -62,6 +92,7 @@ def addrOfCmd : Cli.Cmd := `[Cli| FLAGS: "ixe" : String; "Path to a serialized `.ixe` env to resolve the name in. Without this, the name is looked up in the compiled-in Lean env (via `loadIxonEnv` → `lookupAddr`)." + "ixes" : String; "Path to a `.ixes` shard manifest (requires --ixe): also report which shard owns the name's check-schedule block — the shard whose prove type-checks this constant." ARGS: name : String; "Fully-qualified Lean.Name to resolve (e.g. `Nat.add_comm` or `Tests.Ix.Kernel.TutorialDefs.basicDef`)." diff --git a/Ix/Cli/BenchCmd.lean b/Ix/Cli/BenchCmd.lean index b54814f9..4076d8da 100644 --- a/Ix/Cli/BenchCmd.lean +++ b/Ix/Cli/BenchCmd.lean @@ -38,6 +38,7 @@ module public import Cli public import Lean.Data.Json public import Ix.Benchmark.Results +public import Ix.Cli.CheckCmd public import Ix.Cli.ConstsFile public section @@ -131,6 +132,10 @@ structure EnvSpec where module : String def envSpecs : List EnvSpec := [ + -- Init is the Aiur shard pipeline's deliverable env (the partition the + -- full-Init proof runs over) — registered on its own even though + -- InitStd's closure contains it. + { name := "Init", module := "Benchmarks/Compile/CompileInit.lean" }, { name := "InitStd", module := "Benchmarks/Compile/CompileInitStd.lean" }, { name := "Lean", module := "Benchmarks/Compile/CompileLean.lean" }, { name := "Mathlib", module := "Benchmarks/Compile/CompileMathlib.lean" }, @@ -177,6 +182,13 @@ structure BackendSpec where defaultMode : String /-- The inputs (envs and row names) this backend's runs fan over. -/ inputs : BenchInputs + /-- (mode, envs): the envs whose runs in this mode ALSO measure one + whole-env row, keyed by the env name, next to the per-constant + rows (aiur execute: `ix shard .ixe --max-ram 500` — a + whole-env execution that additionally tracks the predicted fleet + partition). List an env only where the whole-env run fits the CI + host in that mode. -/ + envRows : List (String × List String) := [] /-- `some reason` ⇒ `parse` skips the backend with the note in the config summary. -/ disabled : Option String := none @@ -230,11 +242,28 @@ def backendSpecs : List BackendSpec := [ ("execute", "aiur-check-execute-x64-32x"), ("recursive", "aiur-check-recursive-x64-32x")], unscheduled := ["recursive"], + -- The execute run also measures a whole-env row per envRows env: the + -- FULL check schedule through the codegen'd kernel via `ix shard`'s + -- measured scan — the regime the single-constant rows never enter (a + -- per-constant run faults a tiny closure). The scan IS a whole-env + -- execution; the cut on top is an in-memory merge over the collected + -- block records, so `execute-time` stays an execution wall while the + -- row also tracks the partition predicted for the prove fleet at the + -- pinned `aiurShardBudgetGb`: the shard count (plotted — a packer or + -- kernel win is the only legitimate mover) and the env's total + -- measured fft-cost, which joins the per-constant series on the + -- "Aiur FFT Cost" plot (the execute testbed carries the canonical + -- fft trend; the prove-side duplicate is the unplotted copy). + -- InitStd and Lean + -- only: the heaviest envs whose execute-only wall fits a CI job + -- (Mathlib/FLT hit the dense-core multi-hour execution floor), while + -- Init's execution is contained in InitStd's. + envRows := [("execute", ["InitStd", "Lean"])], metrics := [("prove", ["prove-time", "throughput", "peak-rss", "execute-time", "verify-time", "proof-size", "fft-cost"]), ("execute", ["execute-time", "throughput", "peak-rss", - "fft-cost"]), + "fft-cost", "shards"]), ("recursive", ["recursive-prove-time", "recursive-peak-rss", "recursive-proof-size", "recursive-verify-time", "recursive-execute-time", "recursive-fft-cost", @@ -243,9 +272,15 @@ def backendSpecs : List BackendSpec := [ -- upper-only 5% instead of a hard pin. peak-rss and throughput are -- phase-scoped by the CELL (execute vs prove testbed); -- prove/verify-time and proof-size exist only on the prove testbed. + -- shards exists only on the whole-env execute rows: shard count can + -- only legitimately drop (a packer or kernel win), so it pins upper + -- at 0. (The heaviest shard's own cost is NOT tracked: any repack + -- reshapes it, so a band on it alerts on shard shape, not + -- regressions.) thresholds := [("constants", "0", "0"), ("fft-cost", "0.05", "_"), ("prove-time", "0.10", "_"), ("verify-time", "0.10", "_"), ("proof-size", "0.05", "_"), ("execute-time", "0.10", "_"), + ("shards", "0", "_"), ("peak-rss", "0.10", "_"), ("throughput", "_", "0.10")] }, -- The aiur-recursive run (bench-typecheck --recursive over -- `recursiveConstants`): IxVM recursion on real statements — prove each @@ -356,12 +391,25 @@ def findBackend (name : String) : Option BackendSpec := def recursiveConstants : List String := ["Nat.add_comm"] +/-- The whole-env execute rows' pinned packing budget (GiB): `ix shard + .ixe --max-ram` for the partition the row tracks. Machine- + independent on purpose — a PR row and its bencher baseline must + describe the same partition problem, so the budget is the deliverable + 500 GB prove-fleet box, NOT the runner's own RAM. The budget is only + the cut threshold: the scan executes with a pool sized to the + runner's actual RAM (`--ceiling-gb` still guards it). -/ +def aiurShardBudgetGb : Nat := 500 + def BackendSpec.testbedFor (b : BackendSpec) (mode : String) : Option String := (b.testbeds.find? (·.1 == mode)).map (·.2) def BackendSpec.metricsFor (b : BackendSpec) (mode : String) : List String := ((b.metrics.find? (·.1 == mode)).map (·.2)).getD [] +/-- The envs whose runs in `mode` measure a whole-env row (`envRows`). -/ +def BackendSpec.envRowEnvs (b : BackendSpec) (mode : String) : List String := + ((b.envRows.find? (·.1 == mode)).map (·.2)).getD [] + /-- `thresholds` rendered as the bencher-track action's `--threshold-*` flags, one percentage-test triple per measure. `__WINDOW__` is the action's placeholder for the per-workload baseline window (data points @@ -386,7 +434,8 @@ def BackendSpec.scheduledModes (b : BackendSpec) : List String := constant excluded from one mode (e.g. prove) still keeps its env if another scheduled mode (e.g. execute) runs it; the fixed-config backend is env-independent, pinned to the head env so CI schedules - exactly one entry. -/ + exactly one entry. An env with a whole-env row (`envRows`) in a + scheduled mode is covered like one with selected constants. -/ def BackendSpec.envNames (b : BackendSpec) (rows : Array VectorRow) : List String := let names := envSpecs.map (·.name) @@ -395,6 +444,7 @@ def BackendSpec.envNames (b : BackendSpec) (rows : Array VectorRow) : | .perConstant | .perConstantWithEnv => names.filter fun env => b.scheduledModes.any fun m => + (b.envRowEnvs m).contains env || !(selectNames rows env b.name m (full := false) (tier := "") (shardOnly := false)).isEmpty | .fixedConfigs => names.take 1 @@ -414,7 +464,8 @@ def BackendSpec.benchmarkNames (b : BackendSpec) (rows : Array VectorRow) | .perConstant | .perConstantWithEnv => let mut ns : Array String := #[] for env in b.envNames rows do - if b.inputs == .perConstantWithEnv then ns := ns.push env + if b.inputs == .perConstantWithEnv + || (b.envRowEnvs mode).contains env then ns := ns.push env ns := ns ++ (selectNames rows env b.name mode (full := false) (tier := "") (shardOnly := false)).map (·.name) return ns @@ -607,6 +658,38 @@ def cutClosureShards (ix : String) (envIxe : String) return none return some (subIxe, manifest) +/-- Closure-shard pipeline for aiur heavy-tier constants: extract the + name's dependency closure into a standalone `.ixe`, then cut it with + the MEASURED scan (`ix shard --max-ram`) at the given + budget — no profile step: the scan executes the closure's check + schedule and union-prices the shards. The partition's thin frontiers + are internal to the closure, so checking/proving every shard is an + unconditional verdict on the constant at per-shard RAM instead of + the full-spine re-derivation a standalone `Check` claim pays (473 + GiB measured on bitblast's closure alone). Cached by slug like + [`cutClosureShards`]; `none` ⇒ the caller falls back to a single + leaf. -/ +def cutAiurClosureShards (ix : String) (envIxe : String) + (dir : String) (name : String) (maxRamGb : Nat) : + IO (Option (String × String)) := do + let slug := name.map fun c => + if c == '/' || c == ' ' || c == '.' || c == ':' then '_' else c + let subIxe := s!"{dir}/{slug}.ixe" + let manifest := s!"{dir}/{slug}.ixes" + if (← FilePath.pathExists subIxe) && (← FilePath.pathExists manifest) then + return some (subIxe, manifest) + IO.FS.createDirAll dir + let steps : List (Array String) := + [ #["shard", "extract", envIxe, "--consts", name, "--out", subIxe] + , #["shard", subIxe, "--max-ram", toString maxRamGb, "--out", manifest] ] + for args in steps do + let exit ← runGuarded none 0 ix args + if exit != 0 then + IO.eprintln s!"[bench] closure-shard pipeline failed for '{name}' \ + (exit {exit}); falling back to single leaf" + return none + return some (subIxe, manifest) + /-- Final run gate from the rows themselves: exit 1 when any EXPECTED name lacks a row (an aborted loop, a killed batch, or a dropped whole-env check must never look green — every selected name owes exactly one @@ -778,6 +861,49 @@ def runBenchRunCmd (p : Cli.Parsed) : IO UInt32 := do IO.eprintln s!"[bench] per-constant closures failed (exit {exit})" | "aiur" => let ixe ← ensureIxe repo info ((p.flag? "ixe").map (·.as! String)) + -- Whole-env row (keyed by the env name) on the mode's `envRows` envs: + -- the FULL check schedule through the codegen'd kernel via `ix + -- shard`'s measured scan, which cuts the partition predicted for the + -- prove fleet at `aiurShardBudgetGb` on the way (an in-memory merge + -- over the block records the execution already collected — no second + -- execution, no re-price). The tool self-reports execute-time / + -- peak-rss through the rows contract (env load excluded from the + -- timed window, RSS = process-tree high-water, covering the worker + -- pool); shards, max-shard-fft, and the env's total fft-cost come + -- from the manifest. A kill (≥128) records `oom`/`crash` — the + -- honest row for a box the env no longer fits; a kernel reject exits + -- nonzero with no row and the gate fails on the missing row. + -- Skipped under a `--consts` override — a targeted run stays + -- targeted. + if wanted.isEmpty && (spec.envRowEnvs mode).contains env then + let ix ← resolveBin repo "ix" + let manifest := s!"{repo}/{env}-shardbench.ixes" + let exit ← runGuarded watchdog ceilingGb ix + #["shard", ixe, "--max-ram", toString aiurShardBudgetGb, + "--out", manifest, "--json", out, "--json-name", info.name] + if exit ≥ 128 then + let status := killStatus exit + IO.eprintln s!"[bench] whole-env scan killed (exit {exit}); recording {status}" + markKilled out info.name status + else if exit != 0 then + IO.eprintln s!"[bench] ix shard scan failed (exit {exit})" + else + -- Partition metrics straight from the manifest's tagged costs + -- (measured fft): count and total — the deterministic trend + -- lines a kernel or packer change moves. + match Ix.Cli.CheckCmd.parseIxesShards (← IO.FS.readBinFile manifest) with + | .error e => IO.eprintln s!"[bench] manifest parse failed: {e}" + | .ok shardRows => + if shardRows.isEmpty then + IO.eprintln s!"[bench] empty partition in {manifest}" + else + let totalFft := shardRows.foldl (fun s r => s + r.cost) 0 + let rows ← readRows out + if let some envRow := (rows.getObjVal? info.name).toOption then + writeEntry out info.name <| + (envRow.setObjVal! "shards" + (Lean.toJson shardRows.size)).setObjVal! + "fft-cost" (Lean.toJson totalFft) let bt ← resolveBin repo "bench-typecheck" let modeArgs := match mode with | "execute" => #["--execute-only"] @@ -787,10 +913,85 @@ def runBenchRunCmd (p : Cli.Parsed) : IO UInt32 := do | "execute" => "execute-time" | "recursive" => "recursive-prove-time" | _ => "prove-time" - runPerConstant out names doneKey fun name => + let leaf := fun name => runGuarded watchdog ceilingGb bt (#["--ixe", ixe, "--consts", name, "--json", out, "--texray"] ++ modeArgs) + -- Heavy tier runs as its closure-shard partition instead of one + -- full-closure leaf: a standalone `Check` claim has no frontier, so + -- the kernel re-derives the constant's whole dependency spine — a + -- record costlier than an entire env shard's (bitblast's closure: + -- 1098 BFFT / 473 GiB proved, vs 348 GiB for the env shard owning + -- it). The closure partition's thin frontiers are internal, so + -- checking/proving all its shards is the same unconditional verdict + -- at per-shard RAM. One sub-row per shard (`/shard-K`); the + -- parent row carries the aggregate (summed time = the serial cost, + -- max RSS, shard count, the manifest's total measured fft). + let heavy := if mode == "recursive" then #[] else + (selected.filter (·.tier == "heavy")).map (·.name) + let light := names.filter (!heavy.contains ·) + runPerConstant out light doneKey leaf + let ix ← resolveBin repo "ix" + for name in heavy do + -- Resume: a complete parent row means every shard already ran. + let rows ← readRows out + if ((rows.getObjVal? name).toOption.bind + fun r => (r.getObjVal? doneKey).toOption).isSome then + continue + match ← cutAiurClosureShards ix ixe s!"{repo}/aiurshards-{env}" name + ceilingGb with + | none => + let _ ← leaf name + | some (subIxe, manifest) => + match Ix.Cli.CheckCmd.parseIxesShards + (← IO.FS.readBinFile manifest) with + | .error e => + IO.eprintln s!"[bench] manifest parse failed for '{name}': {e}" + | .ok shardRows => + let n := shardRows.size + let totalFft := shardRows.foldl (fun s r => s + r.cost) 0 + let sub := if mode == "prove" then "prove" else "check" + let mut greens : Array String := #[] + let mut bad := false + for k in [0:n] do + let rowName := s!"{name}/shard-{k}" + let exit ← runGuarded watchdog ceilingGb ix + #[sub, "--ixe", subIxe, "--ixes", manifest, + "--shard", toString k, "--json", out, + "--json-name", rowName] + if exit == 0 then + greens := greens.push rowName + else if exit ≥ 128 then + let status := killStatus exit + IO.eprintln + s!"[bench] '{rowName}' killed (exit {exit}); recording {status}" + markKilled out rowName status + bad := true + else + IO.eprintln s!"[bench] '{rowName}' failed (exit {exit})" + bad := true + -- The parent row lands only when every shard is green — a + -- partial partition must never look like a complete verdict + -- (the gate then reports the missing parent). + if !bad && greens.size == n then + let rows ← readRows out + let getF (rn key : String) : Option Float := + match ((rows.getObjVal? rn).toOption.bind + fun r => (r.getObjVal? key).toOption) with + | some (.num v) => some v.toFloat + | _ => none + let times := greens.filterMap (getF · doneKey) + let rsss := greens.filterMap (getF · "peak-rss") + let sizes := greens.filterMap (getF · "proof-size") + let mut fields : List (String × Lean.Json) := + [ (doneKey, jsonRound 3 (times.foldl (· + ·) 0.0)) + , ("peak-rss", jsonRound 0 (rsss.foldl max 0.0)) + , ("shards", Lean.toJson n) + , ("fft-cost", Lean.toJson totalFft) ] + if mode == "prove" && sizes.size == n then + fields := fields ++ + [("proof-size", jsonRound 0 (sizes.foldl (· + ·) 0.0))] + writeRow out name "ok" fields | "aiur-recursive" => -- Fixed IxVM statements, resolved in the run's `.ixe`: the same -- spawn as the aiur recursive mode, over `recursiveConstants` @@ -856,7 +1057,11 @@ def runBenchRunCmd (p : Cli.Parsed) : IO UInt32 := do | "decompile" => #[info.name] | "ooc" | "lean4lean" => #[info.name] ++ names | "aiur-recursive" => recursiveConstants.toArray - | _ => names + | _ => + -- An `envRows` env owes its whole-env row too (unless a --consts + -- override targeted the run). + if wanted.isEmpty && (spec.envRowEnvs mode).contains env + then #[info.name] ++ names else names let code ← gate out expected if code == 0 || code == exitRejected then saveBaseline out s!"{backend}-{env}-{mode}" @@ -898,7 +1103,7 @@ def benchRunCmd : Cli.Cmd := `[Cli| "Execute one benchmark run (backend × env × mode), writing benchmark results JSON. Exits 0 on success (rows saved as the local baseline), 3 when the kernel rejected any constant, 1 when no rows were produced." FLAGS: - backend : String; "aiur | zisk | sp1 | ooc | lean4lean | compile | decompile | aiur-recursive" + backend : String; "aiur | aiur-recursive | zisk | sp1 | ooc | lean4lean | compile | decompile" env : String; "Benchmark env from the registry (default: InitStd)" mode : String; "prove | execute | recursive (default: the backend's defaultMode)" out : String; "Benchmark results JSON output path (default: bench.json)" diff --git a/Ix/Cli/BenchPlots.lean b/Ix/Cli/BenchPlots.lean index a53b6e31..eb06e76a 100644 --- a/Ix/Cli/BenchPlots.lean +++ b/Ix/Cli/BenchPlots.lean @@ -64,10 +64,11 @@ def plotTitle (workload measure : String) : String := | "aiur-check-prove", "peak-rss" => "Aiur Prove Peak RAM Usage" | "aiur-check-prove", "verify-time" => "Aiur Verify Time" | "aiur-check-prove", "proof-size" => "Aiur Proof Size" - | "aiur-check-prove", "fft-cost" => "Aiur FFT Cost" + | "aiur-check-execute", "fft-cost" => "Aiur FFT Cost" | "aiur-check-execute", "execute-time" => "Aiur Execute Time" | "aiur-check-execute", "throughput" => "Aiur Execute Throughput" | "aiur-check-execute", "peak-rss" => "Aiur Execute Peak RAM Usage" + | "aiur-check-execute", "shards" => "Aiur Predicted Shards" | "zisk-check-execute", "execute-time" => "Zisk Execute Time" | "zisk-check-execute", "throughput" => "Zisk Execute Throughput" | "zisk-check-execute", "peak-rss" => "Zisk Execute Peak RAM Usage" @@ -85,8 +86,11 @@ def plotTitle (workload measure : String) : String := /-- Tracked but not plotted solo. The two aiur runs re-measure each other's deterministic Phase-1 numbers as a redundancy check — one - trend line each is enough ("Aiur Execute Time" from the execute run, - "Aiur FFT Cost" from the prove run). Zisk `shards` is charted below + trend line each is enough, and both live on the execute testbed + ("Aiur Execute Time", "Aiur FFT Cost"), where the whole-env rows + upload too — so the env series and the per-constant series share + the same plots. The prove run's duplicates (execute-time, fft-cost) + are the skipped copies. Zisk `shards` is charted below over the heavy-tier primaries alone (light constants are pinned at a single shard, a flat line at 1), not over the full set here; zisk `constants` charts on the input-constants plot below instead of alone. @@ -100,7 +104,7 @@ def plotTitle (workload measure : String) : String := recursion layer's own `recursive-*` series, so the inner metrics aren't plotted. -/ def plotSkips : List (String × String) := - [("aiur-check-prove", "execute-time"), ("aiur-check-execute", "fft-cost"), + [("aiur-check-prove", "execute-time"), ("aiur-check-prove", "fft-cost"), ("zisk-check-execute", "shards"), ("zisk-check-execute", "constants"), ("ix-decompile", "file-size"), ("ix-decompile", "constants"), ("aiur-recursive", "prove-time"), ("aiur-recursive", "proof-size"), diff --git a/Ix/Cli/BenchReport.lean b/Ix/Cli/BenchReport.lean index 9a397a8b..2dad7789 100644 --- a/Ix/Cli/BenchReport.lean +++ b/Ix/Cli/BenchReport.lean @@ -822,8 +822,11 @@ def runMatrixCmd (p : Cli.Parsed) : IO UInt32 := do -- an env stays in `envNames` because some scheduled mode runs it, -- but a constant excluded from one mode (e.g. Lean's only primary -- constant is prove-excluded) leaves that (env, mode) cell empty — - -- scheduling it would waste a job and expect a row that never lands. + -- scheduling it would waste a job and expect a row that never + -- lands. A whole-env row (`envRows`) keeps the cell scheduled + -- even with no per-constant selection. if b.inputs == .perConstant + && !(b.envRowEnvs mode).contains env && (Ix.Cli.BenchCmd.selectNames rows env b.name mode (full := false) (tier := "") (shardOnly := false)).isEmpty then continue @@ -868,7 +871,8 @@ def parseError (msg : String) : IO UInt32 := do Grammar (an unknown command-line token, or an unknown env in BENCH_ENVS, rejects the command — exit 2 and a `parse-error` output): - !benchmark ([aiur] [zisk] [sp1] [ooc] [compile] [aiur-recursive] | all) + !benchmark ([aiur] [aiur-recursive] [zisk] [sp1] [ooc] [compile] + [decompile] | all) [execute | recursive] [fresh] [KEY=VALUE …] BENCH_ENVS=InitStd,Mathlib (case-insensitive, any registry env; defaults to every env for the diff --git a/Ix/Cli/CheckCmd.lean b/Ix/Cli/CheckCmd.lean index 56553878..861f902b 100644 --- a/Ix/Cli/CheckCmd.lean +++ b/Ix/Cli/CheckCmd.lean @@ -24,6 +24,7 @@ public import Ix.Aiur.Interpret public import Ix.Aiur.Protocol public import Ix.Aiur.Statistics public import Ix.AssumptionTree +public import Ix.Benchmark.Results public import Ix.Claim public import Ix.Common public import Ix.IxVM @@ -32,6 +33,7 @@ public import Ix.IxVM.ClaimHarness public import Ix.Ixon public import Ix.Meta public import Ix.Store +public import Ix.TracingTexray public import Ix.Cli.NameResolve public section @@ -363,38 +365,66 @@ private def ixesAddr : IxesP Address := do if p + 32 ≤ b.size then modify (fun _ => (b, p + 32)); pure ⟨b.extract p (p + 32)⟩ else throw "ixes: truncated (expected a 32-byte address)" -/-- Parse every shard's owned block addresses from a serialized `.ixes` - manifest (`ShardManifest::to_bytes`, `src/ix/shard.rs`): - magic(8) ‖ total_cross_ingress(u128) ‖ num_shards(u32) ‖ per shard - { id(u32) ‖ heartbeats(u64) ‖ own_size(u64) ‖ cross_ingress(u64) ‖ - assumption_root(u8 tag + 32?) ‖ blocks(u32 len + 32·len) ‖ - foreign_blocks(u32 len + 32·len) }. +private def ixesU64 : IxesP Nat := do + let mut v : Nat := 0 + for i in [0:8] do + v := v ||| ((← ixesU8).toNat <<< (8 * i)) + pure v + +/-- One shard row of a parsed `.ixes` manifest: the owned block addresses + plus the planner cost (`ShardCost` in `crates/kernel/src/shard.rs` — + `costTag` 0 = unknown, 1 = profile heartbeats, 2 = Zisk cost units, + 3 = Aiur fft; `cost` is the scalar, comparable within one manifest). -/ +structure IxesShard where + blocks : Array Address + costTag : UInt8 + cost : Nat + +/-- Parse every shard of a serialized `.ixes` manifest + (`ShardManifest::to_bytes`, `crates/kernel/src/shard.rs`, format v2): + magic("IXES\0\0\0" ++ version) ‖ total_cross_ingress(u128) ‖ + num_shards(u32) ‖ per shard + { id(u32) ‖ cost_tag(u8) ‖ cost(u64) ‖ own_size(u64) ‖ + cross_ingress(u64) ‖ assumption_root(u8 tag + 32?) ‖ + blocks(u32 len + 32·len) ‖ foreign_blocks(u32 len + 32·len) }. Bounds-checked: a truncated/malformed file yields `.error`, never a panic. -/ -def parseIxesAllShards (bytes : ByteArray) : Except String (Array (Array Address)) := - let go : IxesP (Array (Array Address)) := do +def parseIxesShards (bytes : ByteArray) : Except String (Array IxesShard) := + let go : IxesP (Array IxesShard) := do let m0 ← ixesU8; let m1 ← ixesU8; let m2 ← ixesU8; let m3 ← ixesU8 if !(m0 == 0x49 && m1 == 0x58 && m2 == 0x45 && m3 == 0x53) then throw "not an .ixes file (bad magic)" - ixesSkip 4 -- rest of the 8-byte magic + ixesSkip 3 -- reserved zero bytes of the 8-byte magic + let version ← ixesU8 + if version != 2 then + throw s!"unsupported .ixes format version {version} (expected 2) — \ + regenerate the manifest with the current `ix shard`" ixesSkip 16 -- total_cross_ingress (u128) let n ← ixesU32 - let mut shards : Array (Array Address) := #[] + let mut shards : Array IxesShard := #[] for _ in [0:n.toNat] do - ixesSkip (4 + 8 + 8 + 8) -- id + heartbeats + own_size + cross_ingress + ixesSkip 4 -- id + let costTag ← ixesU8 + let cost ← ixesU64 + ixesSkip (8 + 8) -- own_size + cross_ingress if (← ixesU8) == 1 then ixesSkip 32 -- assumption_root present let blen ← ixesU32 let mut blocks : Array Address := #[] for _ in [0:blen.toNat] do blocks := blocks.push (← ixesAddr) ixesSkip ((← ixesU32).toNat * 32) -- skip foreign_blocks - shards := shards.push blocks + shards := shards.push { blocks, costTag, cost } pure shards go.run' (bytes, 0) +/-- The owned block addresses of every shard (cost columns dropped). -/ +def parseIxesAllShards (bytes : ByteArray) : Except String (Array (Array Address)) := + (parseIxesShards bytes).map (·.map (·.blocks)) + /-- The check-schedule block address of a constant: a projection collapses to its SCC/Muts wrapper (`p.block`); everything else is its own block. - Mirrors `check_schedule_block_addr` (`src/ffi/kernel.rs`). -/ -private def blockAddrOf (addr : Address) (c : Ixon.Constant) : Address := + Mirrors `check_schedule_block_addr` (`src/ffi/kernel.rs`). Public for + `ix addr-of --ixes`' owning-shard lookup. -/ +def blockAddrOf (addr : Address) (c : Ixon.Constant) : Address := match c.info with | .iPrj prj => prj.block | .cPrj prj => prj.block @@ -484,7 +514,8 @@ def runShardCheckManifest (manifestPath ixePath : String) (shardK : Nat) once for this one call. -/ def runShardCheckManifestNative (manifestPath ixePath : String) (shardK : Nat) (compiled : Aiur.CompiledToplevel) (printStats : Bool) - (statsOut : Option String) (useBytecode : Bool) : IO UInt32 := do + (statsOut : Option String) (useBytecode : Bool) + (benchJson : Option (String × String) := none) : IO UInt32 := do match (← loadEnvAndShards manifestPath ixePath) with | .error e => IO.eprintln e; return 1 | .ok (ixonEnv, shards) => match shards[shardK]? with @@ -493,7 +524,26 @@ def runShardCheckManifestNative (manifestPath ixePath : String) (shardK : Nat) let envHandle ← match Aiur.EnvHandle.fromIxe ixePath with | .error e => IO.eprintln s!"EnvHandle.fromIxe {ixePath}: {e}"; return 1 | .ok h => pure h - runShardOwnedNative envHandle compiled printStats statsOut useBytecode ixonEnv blocks shardK + -- `benchJson = (out, rowName)` reports the check as a benchmark + -- row: `execute-time` windows the check itself — the env parse and + -- `EnvHandle` build are excluded, so the measure tracks the + -- kernel, not the loader — while `peak-rss` is the process tree's + -- absolute high-water (the parsed env sits in the baseline, + -- matching the ooc rows' semantics). + if benchJson.isSome then + TracingTexray.startSampler + TracingTexray.resetPeakTreeRss + let start ← IO.monoMsNow + let rc ← runShardOwnedNative envHandle compiled printStats statsOut + useBytecode ixonEnv blocks shardK + if let some (out, rowName) := benchJson then + if rc == 0 then + let secs := ((← IO.monoMsNow) - start).toFloat / 1000.0 + let peakRss ← TracingTexray.peakTreeRssBytes + Ix.Benchmark.Results.writeRow out rowName "ok" + [ ("execute-time", Ix.Benchmark.Results.jsonRound 3 secs) + , ("peak-rss", Lean.toJson peakRss) ] + return rc /-- Coverage check over already-loaded env + shards: every constant's check-schedule block is owned by **exactly one** shard. That is the whole @@ -617,11 +667,61 @@ def runCheckCmd (p : Cli.Parsed) : IO UInt32 := do | some other => IO.eprintln s!"error: --interp expects \"source\" or \"bytecode\", got \"{other}\"" return 1 - let keepGoing := p.hasFlag "keep-going" + let keepGoing := p.hasFlag "no-fail-fast" + if keepGoing && p.hasFlag "fail-fast" then + p.printError "error: --fail-fast and --no-fail-fast are mutually exclusive" + return 1 let statsOut : Option String := (p.flag? "stats-out").map (·.as! String) let ixePath : Option String := (p.flag? "ixe").map (·.as! String) + if p.hasFlag "execute" then + -- Execute-only whole-env check: the Aiur-kernel counterpart of the + -- Rust kernel's whole-env check — parallel execution of every + -- block's claim, no partition, no manifest, no prove concerns. + let some ixe := ixePath | do + p.printError "error: --execute requires --ixe" + return 1 + let toplevel ← match IxVM.ixVM with + | .error e => IO.eprintln s!"Toplevel merging failed: {e}"; return 1 + | .ok t => pure t + let compiled ← match toplevel.compile with + | .error e => IO.eprintln s!"Compilation failed: {e}"; return 1 + | .ok c => pure c + let funIdx ← match compiled.getFuncIdx `verify_claim with + | some i => pure i + | none => IO.eprintln "error: verify_claim missing"; return 1 + let envHandle ← match Aiur.EnvHandle.fromIxe ixe with + | .error e => IO.eprintln s!"EnvHandle.fromIxe {ixe}: {e}"; return 1 + | .ok h => pure h + let workers := (p.flag? "jobs").map (·.as! Nat) |>.getD 0 + let workerBin := (← IO.appPath).toString + -- `--json` reports the run as a benchmark row: `execute-time` + -- windows the parallel check itself — the kernel compile and + -- `EnvHandle` build are excluded, so the measure tracks the kernel, + -- not the loader — while `peak-rss` is the process tree's absolute + -- high-water (covers the worker pool). + let benchJson := (p.flag? "json").map fun f => + (f.as! String, + ((p.flag? "json-name").map (·.as! String)).getD "execute") + if benchJson.isSome then + TracingTexray.startSampler + TracingTexray.resetPeakTreeRss + let start ← IO.monoMsNow + match Aiur.Bytecode.Toplevel.executeEnvWithEnv compiled.bytecode funIdx + envHandle (toString workers) (if keepGoing then "0" else "1") + workerBin ixe with + | .error e => IO.eprintln s!"execute failed: {e}"; return 1 + | .ok () => + let ms := (← IO.monoMsNow) - start + IO.println s!"execute: OK in {ms} ms" + if let some (out, rowName) := benchJson then + let peakRss ← TracingTexray.peakTreeRssBytes + Ix.Benchmark.Results.writeRow out rowName "ok" + [ ("execute-time", + Ix.Benchmark.Results.jsonRound 3 (ms.toFloat / 1000.0)) + , ("peak-rss", Lean.toJson peakRss) ] + return 0 let claimHex : Option String := (p.flag? "claim").map (·.as! String) let names := (p.variableArgsAs! String).toList @@ -672,7 +772,11 @@ def runCheckCmd (p : Cli.Parsed) : IO UInt32 := do let compiled ← match toplevel.compile with | .error e => IO.eprintln s!"Compilation failed: {e}"; return 1 | .ok c => pure c - return (← runShardCheckManifestNative manifest ixe k compiled printStats statsOut useBytecode) + let benchJson := (p.flag? "json").map fun f => + (f.as! String, + ((p.flag? "json-name").map (·.as! String)).getD s!"shard-{k}") + return (← runShardCheckManifestNative manifest ixe k compiled printStats + statsOut useBytecode benchJson) | some ixe, some manifest, none => if interpSource then return (← runShardCheckAll manifest ixe ((p.flag? "jobs").map (·.as! Nat)) @@ -695,13 +799,17 @@ def checkCmd : Cli.Cmd := `[Cli| FLAGS: interp : String; "Use an interpreter instead of the codegen'd IxVM Rust kernel. Modes: `source` = Aiur source interpreter (richer per-execution error diagnostics, slowest); `bytecode` = generic Aiur bytecode interpreter (skips the regen + cargo rebuild cycle when iterating on `Ix/IxVM/*.lean`). Omit the flag entirely for the native codegen kernel." - "keep-going"; "Continue past failures and report them at the end instead of halting on the first." + "fail-fast"; "Halt on the first failure (the default; flag accepted for explicitness)." + "no-fail-fast"; "Continue past failures and report them at the end instead of halting on the first." "ixe" : String; "Path to a serialized `.ixe` env. When set, the binary reads the env from disk instead of using the compiled-in Lean env." + "execute"; "Execute-only whole-env check (requires --ixe): run every block's check claim through the codegen'd Aiur kernel in parallel — no partition, no manifest, no proving. Reports blocks checked, kernel rejects (named), and total measured FFT cost. --jobs bounds the worker count (default: autoscale); combine with --no-fail-fast to inventory every reject." "claim" : String; "32-byte hex address of a persisted `Ix.Claim` in `~/.ix/store/`. When set, runs the `verify_claim` entrypoint once over the claim's witness against the `--ixe` env (single execution, skips per-const iteration)." "stats-out" : String; "Redirect the per-circuit statistics dump to this file (only used when exactly one constant is targeted)." "ixes" : String; "Path to a `.ixes` shard manifest (with --ixe). With --shard K: check the constants owned by shard K (ingress their closure, skip the frontier). Without --shard: check every shard of the partition concurrently, after a coverage check." "shard" : Nat; "0-based shard index K (with --ixe + --ixes): check the constants owned by shard K of the manifest's partition." "jobs" : Nat; "Max shards to check concurrently when checking a whole partition (--ixes without --shard). Default: all at once. Lower it to bound peak RAM — each in-flight shard re-ingests its closure into its own IO buffer." + json : String; "Benchmark results JSON accumulator (single-shard and --execute modes): append an `execute-time`/`peak-rss` row for the checked shard or whole-env execute." + "json-name" : String; "Row name for --json (default: shard-, or `execute` for --execute)." ARGS: ...names : String; "Fully-qualified Lean.Name(s) to check. With none, iterate every named constant in the env (sorted)." diff --git a/Ix/Cli/NameOfCmd.lean b/Ix/Cli/NameOfCmd.lean index 1a5e80d8..074a3647 100644 --- a/Ix/Cli/NameOfCmd.lean +++ b/Ix/Cli/NameOfCmd.lean @@ -60,22 +60,86 @@ def nameLookup (ixonEnv : Ixon.Env) (addr : Address) : IO UInt32 := do return 1 return 0 +/-- Batch resolution: one env decode + one pass building both reverse + indexes (address → names, block → projection names), then O(1) per + address — the single-address path's per-call full scans cost ~90 s + each at FLT scale, which forbids inventories of tens of thousands. + Each address resolves to `[,…]` (`prj:` prefix for + projection fallbacks into unnamed blocks, `` for + misses). -/ +def resolveAddrs (ixonEnv : Ixon.Env) (addrs : Array Address) : + Array (Address × String) := Id.run do + let mut byAddr : Std.HashMap Address (Array String) := {} + for (n, named) in ixonEnv.named do + byAddr := byAddr.insert named.addr + ((byAddr.getD named.addr #[]).push (toString (ixNameToLeanName n))) + let mut byBlock : Std.HashMap Address (Array String) := {} + for (caddr, lc) in ixonEnv.consts do + let some c := lc.get? | continue + let blk? := match c.info with + | .iPrj p => some p.block + | .cPrj p => some p.block + | .rPrj p => some p.block + | .dPrj p => some p.block + | _ => none + if let some blk := blk? then + let nm := match ixonEnv.getName? caddr with + | some n => toString (ixNameToLeanName n) + | none => s!"" + byBlock := byBlock.insert blk ((byBlock.getD blk #[]).push nm) + let mut out : Array (Address × String) := #[] + for a in addrs do + let disp := match byAddr.get? a with + | some ns => String.intercalate "," ns.toList + | none => match byBlock.get? a with + | some ns => s!"prj:{String.intercalate "," ns.toList}" + | none => "" + out := out.push (a, disp) + return out + +def batchLookup (ixonEnv : Ixon.Env) (addrs : Array Address) : IO UInt32 := do + let resolved := resolveAddrs ixonEnv addrs + let mut missing := 0 + for (a, disp) in resolved do + IO.println s!"{a} {disp}" + if disp == "" then + missing := missing + 1 + if missing > 0 then + IO.eprintln s!"[name-of] {missing} address(es) unresolved" + return 0 + def runNameOfCmd (p : Cli.Parsed) : IO UInt32 := do - let some addrArg := p.positionalArg? "addr" - | p.printError "error: must specify a 64-char hex address"; return 1 - let argStr := addrArg.as! String - let some addr := Address.fromString argStr - | IO.eprintln s!"error: `{argStr}` is not a 64-char hex address" - return 1 let some path := (p.flag? "ixe").map (·.as! String) | IO.eprintln "error: name-of requires --ixe " return 1 + let batchFile := (p.flag? "addrs-file").map (·.as! String) + let addrArgs := p.variableArgsAs! String + if batchFile.isNone && addrArgs.isEmpty then + p.printError "error: pass a 64-char hex address or --addrs-file" + return 1 let bytes ← IO.FS.readBinFile path let ixonEnv ← match Ixon.deEnvAnon bytes with | .error e => IO.eprintln s!"error: failed to deserialize {path}: {e}"; return 1 | .ok env => pure env - nameLookup ixonEnv addr + match batchFile with + | some file => + let mut addrs : Array Address := #[] + for line in (← IO.FS.readFile file).splitOn "\n" do + let line := line.trimAscii.toString + if line.isEmpty || line.startsWith "#" then + continue + let some a := Address.fromString line + | IO.eprintln s!"error: `{line}` is not a 64-char hex address" + return 1 + addrs := addrs.push a + batchLookup ixonEnv addrs + | none => + let argStr := addrArgs[0]! + let some addr := Address.fromString argStr + | IO.eprintln s!"error: `{argStr}` is not a 64-char hex address" + return 1 + nameLookup ixonEnv addr end Ix.Cli.NameOfCmd @@ -85,10 +149,11 @@ def nameOfCmd : Cli.Cmd := `[Cli| "Resolve a content address back to its Lean name(s) in a `.ixe` env (may print several: structurally equivalent constants share an address)" FLAGS: - "ixe" : String; "Path to a serialized `.ixe` env to resolve the address in (required)." + "ixe" : String; "Path to a serialized `.ixe` env to resolve the address in (required)." + "addrs-file" : String; "Batch mode: file of 64-char hex addresses (one per line; `#` comments and blanks ignored). One env decode resolves them all — output ` [,…]` per line, `prj:` prefix for projection fallbacks." ARGS: - addr : String; "64-char hex content address to resolve. Prints every Lean.Name registered for it, one per line; for unnamed Muts blocks, prints the names of projection constants into the block instead." + ...addr : String; "64-char hex content address to resolve (omit when using --addrs-file). Prints every Lean.Name registered for it, one per line; for unnamed Muts blocks, prints the names of projection constants into the block instead." ] end diff --git a/Ix/Cli/ProfileCmd.lean b/Ix/Cli/ProfileCmd.lean index d42b428b..aeed2e1c 100644 --- a/Ix/Cli/ProfileCmd.lean +++ b/Ix/Cli/ProfileCmd.lean @@ -21,6 +21,38 @@ open Ix.KernelCheck namespace Ix.Cli.ProfileCmd +def runSweepCmd (p : Cli.Parsed) : IO UInt32 := do + let some pathArg := p.positionalArg? "path" + | p.printError "error: must specify to a .ixe file" + return 1 + let envPath := pathArg.as! String + let base := if envPath.endsWith ".ixe" then (envPath.dropEnd 4).toString else envPath + let profPath := (p.flag? "prof").map (·.as! String) |>.getD (base ++ ".ixprof") + let csvPath := (p.flag? "out").map (·.as! String) |>.getD (base ++ "-sweep.csv") + let budget := (p.flag? "budget").map (·.as! Nat) |>.getD 64 + let topBlocks := (p.flag? "top-blocks").map (·.as! Nat) |>.getD 10 + let reps := (p.flag? "reps").map (·.as! Nat) |>.getD 10 + IO.println s!"Sweeping {envPath} × {profPath} → {csvPath} (budget {budget} GiB)" + rsProfileSweepFFI envPath profPath csvPath (toString budget) + (toString topBlocks) (toString reps) + IO.println s!"[sweep] wrote {csvPath}" + return 0 + +def sweepCmd : Cli.Cmd := `[Cli| + "sweep" VIA runSweepCmd; + "Closure cost sweep: predicted Aiur execute/prove cost for every named constant's dependency closure, plus feasibility/bottleneck/diversity reports" + + FLAGS: + prof : String; "Input .ixprof from `ix profile` (default: .ixprof)" + out : String; "Output CSV path (default: -sweep.csv)" + budget : Nat; "RAM budget in GiB for the feasibility report (default 64)" + "top-blocks" : Nat; "Expensive blocks tracked for the min-root report (default 10, max 64)" + reps : Nat; "Diverse feature-mix representatives to pick (default 10; 0 disables)" + + ARGS: + path : String; "Path to the serialized .ixe the .ixprof was profiled from" +] + def runProfileCmd (p : Cli.Parsed) : IO UInt32 := do let some pathArg := p.positionalArg? "path" | p.printError "error: must specify to a .ixe file" @@ -44,9 +76,15 @@ def runProfileCmd (p : Cli.Parsed) : IO UInt32 := do return 1 Std.Internal.UV.System.osSetenv "IX_KERNEL_CHECK_WORKERS" (toString n) + let top := (p.flag? "top").map (·.as! Nat) |>.getD 10 + let backend := (p.flag? "backend").map (·.as! String) |>.getD "all" + if backend != "all" && backend != "aiur" && backend != "zisk" then + p.printError s!"error: --backend must be all, aiur, or zisk (got {backend})" + return 1 + IO.println s!"Profiling {envPath} → {outPath} (isolate={isolate})" let start ← IO.monoMsNow - rsProfileAnonFFI envPath outPath isolate quiet + rsProfileAnonFFI envPath outPath isolate quiet (toString top) backend let elapsed := (← IO.monoMsNow) - start IO.println s!"[profile] wrote {outPath} in {elapsed.formatMs}" return 0 @@ -63,9 +101,14 @@ def profileCmd : Cli.Cmd := `[Cli| "keep-caches"; "Keep cross-constant caches: faster, lower-fidelity, may under-record" workers : Nat; "Parallel kernel workers (default: available_parallelism). Plumbs IX_KERNEL_CHECK_WORKERS." verbose; "Log every constant (default: quiet)" + top : Nat; "Block-leaderboard size in the summary: top N by heartbeats, substitutions, ingress bytes, and predicted per-backend cost (default 10; 0 disables)" + backend : String; "Cost models in the summary: all (default), aiur, or zisk" ARGS: path : String; "Path to a serialized .ixe environment" + + SUBCOMMANDS: + sweepCmd ] end diff --git a/Ix/Cli/ProveCmd.lean b/Ix/Cli/ProveCmd.lean index a35609d9..bfafa2b5 100644 --- a/Ix/Cli/ProveCmd.lean +++ b/Ix/Cli/ProveCmd.lean @@ -30,6 +30,7 @@ public import Ix.Aiur.Compiler public import Ix.Aiur.Protocol public import Ix.Claim public import Ix.Cli.CheckCmd +public import Ix.Cli.VerifyCmd public import Ix.Common public import Ix.IxVM public import Ix.IxVM.Toplevel @@ -53,6 +54,26 @@ private def commitmentParameters : Aiur.CommitmentParameters := private def friParameters : Aiur.FriParameters := Aiur.defaultFriParameters +/-- Resolve a keyed cache namespace: `/` when a root is + given (tests use a scratch root to stay hermetic), else the global + `~/.ix/cache/`. Entries are named by claim digest and hold + re-derivable state, so the directory is safe to wipe. -/ +def cacheSubdir (cacheRoot : Option System.FilePath) (ns : String) : + IO System.FilePath := do + match cacheRoot with + | some root => + let d := root / ns + IO.FS.createDirAll d + pure d + | none => StoreIO.toIO (Store.cacheDir ns) + +/-- Atomic keyed-cache write: `/` via temp file + rename. -/ +def writeCacheEntry (dir : System.FilePath) (key content : String) : + IO Unit := do + let tmp := dir / s!"{key}.tmp" + IO.FS.writeFile tmp content + IO.FS.rename tmp (dir / key) + def proveOne (aiurSystem : Aiur.AiurSystem) (compiled : Aiur.CompiledToplevel) (claim : Ix.Claim) @@ -106,7 +127,8 @@ def proveOne (aiurSystem : Aiur.AiurSystem) def runShardProveNative (manifestPath : String) (envHandle : Aiur.EnvHandle) (ixonEnv : Ixon.Env) (shards : Array (Array Address)) (shardK : Nat) (aiurSystem : Aiur.AiurSystem) (compiled : Aiur.CompiledToplevel) - (_printStats : Bool) : IO UInt32 := do + (_printStats : Bool) + (benchJson : Option (String × String) := none) : IO UInt32 := do match shards[shardK]? with | none => IO.eprintln s!"shard {shardK} out of range (0..{shards.size})"; return 1 | some blocks => do @@ -117,12 +139,23 @@ def runShardProveNative (manifestPath : String) (envHandle : Aiur.EnvHandle) let label := s!"shard {shardK}" IO.println s!"Proving {label}" (← IO.getStdout).flush + -- `benchJson = (out, rowName)` reports the prove as a benchmark row + -- (the `aiur` bench backend's heavy-tier closure-shard spawn): + -- `prove-time` windows the FFI prove itself — env parse, manifest + -- load, and `EnvHandle` build are excluded, matching the check rows' + -- loader-free semantics — while `peak-rss` is the process tree's + -- absolute high-water. + if benchJson.isSome then + TracingTexray.startSampler + TracingTexray.resetPeakTreeRss + let start ← IO.monoMsNow let funIdx := compiled.getFuncIdx `verify_claim |>.get! match aiurSystem.shardProveWithEnv funIdx envHandle blob with | .error e => IO.eprintln s!"{label}: shardProveWithEnv error: {e}" return 1 | .ok (claimBytes, proof, _outIO) => + let ms := (← IO.monoMsNow) - start -- Rust returns the canonical CheckEnv claim's wire bytes; deserialize -- back to `Ix.Claim` to persist alongside the proof. Avoids -- recomputing the closure walk + canonical AssumptionTree Lean-side. @@ -135,11 +168,202 @@ def runShardProveNative (manifestPath : String) (envHandle : Aiur.EnvHandle) let wrapper : Ixon.Proof := { claim, proof := proof.toBytes } let proofAddr ← StoreIO.toIO (Store.write (Ixon.Proof.ser wrapper)) IO.println (toString proofAddr) + if let some (out, rowName) := benchJson then + let peakRss ← TracingTexray.peakTreeRssBytes + Ix.Benchmark.Results.writeRow out rowName "ok" + [ ("prove-time", + Ix.Benchmark.Results.jsonRound 3 (ms.toFloat / 1000.0)) + , ("peak-rss", Lean.toJson peakRss) + , ("proof-size", Lean.toJson proof.toBytes.size) ] let _ := manifestPath -- kept for parity with previous signature return 0 +/-- A shard proven, verified, and bound to its reconstructed claim; + persisted as the `shard-proofs/` cache entry. -/ +structure ProofRow where + shard : Nat + claimDigest : Address + proofAddr : Address + +/-- Batched, resumable all-shards prove — the whole-partition behavior of + `prove --ixes` with no `--shard`. + + One `EnvHandle`, one compiled toplevel, and one `AiurSystem` are shared + across every shard (the per-invocation setup is paid once). Progress + persists in the keyed cache `~/.ix/cache/shard-proofs/` (one file per + claim digest, holding the proof's store address): an entry is written + only after the shard's proof both VERIFIES and binds to the shard's + reconstructed `CheckEnv` claim digest, so a crash, OOM, or interrupt + costs only the in-flight shard and re-running the same command + resumes. The claim digest IS the staleness test — a repacked shard has + a new digest, which never matches an old entry — and it is + manifest-independent, so two manifests sharing a shard share its + proof. `cacheRoot` overrides `~/.ix/cache` (tests). + + `jobs` shards prove concurrently (default 1: each prove peaks at the + shard's full predicted RAM, so concurrency is a big-box knob). + + Ends with the composed verdict: disjoint cover + every shard bound to + a verified proof — the same statement `ix verify --ixe --ixes ` + checks after the fact. -/ +def runShardProveAllNative (manifestPath : String) (envHandle : Aiur.EnvHandle) + (ixonEnv : Ixon.Env) (shards : Array (Array Address)) + (aiurSystem : Aiur.AiurSystem) (compiled : Aiur.CompiledToplevel) + (jobs : Nat) (cacheRoot : Option System.FilePath := none) : IO UInt32 := do + if !(← Ix.Cli.CheckCmd.shardsCover ixonEnv shards) then return 1 + -- Reconstructed claim digest per shard: the binding target for proofs + -- and the cache key. + let mut digests : Array Address := #[] + for blocks in shards do + match Ix.Cli.CheckCmd.shardClaimDigest ixonEnv blocks with + | .error e => IO.eprintln s!"reconstruct shard {digests.size} claim failed: {e}"; return 1 + | .ok d => digests := digests.push d + let proofsDir ← cacheSubdir cacheRoot "shard-proofs" + -- Resume: a shard whose digest names a cache entry is done IF the + -- recorded proof still verifies — the digest pins the claim but not + -- the circuit version (any kernel change between sessions regenerates + -- the codegen and the verifying key), so trusting the entry alone + -- would count a stale-circuit proof. + let mut done : Std.HashMap Nat Address := {} + for k in [0:shards.size] do + let entry := proofsDir / toString digests[k]! + if ← entry.pathExists then + match Address.fromString (← IO.FS.readFile entry).trimAscii.toString with + | some pa => + if (← Ix.Cli.VerifyCmd.verifyOneProof aiurSystem compiled pa) == 0 then + done := done.insert k pa + else + IO.println s!"[prove] shard {k}: cached proof {pa} no longer \ + verifies (circuit changed?) — re-proving" + | none => pure () + let pending := (List.range shards.size).filter (fun k => !done.contains k) + -- Largest-cost first, straight from the manifest's per-shard tagged + -- cost (measured Mfft on scan manifests, model-predicted on packer + -- manifests). If any shard is going to breach the watchdog it is one + -- of the heaviest — proving those first surfaces a failure in the + -- opening minutes instead of hours in, and everything after the heavy + -- head is strictly safer than what already passed. A manifest without + -- costs (all-unknown) degrades to manifest order. + let pending : List Nat ← do + match Ix.Cli.CheckCmd.parseIxesShards (← IO.FS.readBinFile manifestPath) with + | .error e => + IO.println s!"[prove] manifest cost parse failed ({e}); proving in manifest order" + pure pending + | .ok rows => + pure <| pending.mergeSort (fun a b => + (rows[a]?.map (·.cost)).getD 0 ≥ (rows[b]?.map (·.cost)).getD 0) + IO.println s!"[prove] {shards.size} shards: {done.size} already proven \ + (cache {proofsDir}), {pending.length} pending (heaviest first)" + let recordProof (r : ProofRow) : IO Unit := + writeCacheEntry proofsDir (toString r.claimDigest) s!"{r.proofAddr}\n" + let funIdx := compiled.getFuncIdx `verify_claim |>.get! + let proveOneShard (k : Nat) : IO (Except String ProofRow) := do + let blocks := shards[k]! + let blob := Id.run do + let mut b := ByteArray.empty + for a in Ix.Cli.CheckCmd.ownedConstsForBlocks ixonEnv blocks do + b := b ++ a.hash + pure b + IO.println s!"Proving shard {k}" + (← IO.getStdout).flush + match aiurSystem.shardProveWithEnv funIdx envHandle blob with + | .error e => return .error s!"shardProveWithEnv: {e}" + | .ok (claimBytes, proof, _outIO) => + match Ixon.runGet Ix.Claim.get claimBytes with + | .error e => return .error s!"claim wire-decode failed: {e}" + | .ok claim => + let d := Address.blake3 (Ix.Claim.ser claim) + if digests[k]? != some d then + return .error s!"proved claim {d} does not match reconstructed \ + digest {digests[k]!} — witness and reconstruction disagree" + let _ ← StoreIO.toIO (Store.write (Ix.Claim.ser claim)) + let wrapper : Ixon.Proof := { claim, proof := proof.toBytes } + let proofAddr ← StoreIO.toIO (Store.write (Ixon.Proof.ser wrapper)) + if (← Ix.Cli.VerifyCmd.verifyOneProof aiurSystem compiled proofAddr) != 0 then + return .error s!"proof {proofAddr} failed verification" + return .ok { shard := k, claimDigest := d, proofAddr } + -- RAM-aware admission — fastest wall without OOM: admit the next + -- (heaviest-first) shard whenever the predicted-RSS sum of in-flight + -- proves plus its own fits the box limit. Predictions come from the + -- scan sidecar's analytic per-shard `pred_ram_gib` (validated within + -- a few percent of measured RSS); a shard with no sidecar row is + -- assumed to need the whole limit, degrading to serial for exactly + -- those shards. `--jobs` remains as an optional concurrency ceiling. + let predRam : Std.HashMap Nat Nat ← do + let path : System.FilePath := ⟨manifestPath ++ ".costs.csv"⟩ + let mut m : Std.HashMap Nat Nat := {} + if ← path.pathExists then + for line in ((← IO.FS.readFile path).splitOn "\n").drop 1 do + let cols := line.splitOn "," + if cols.length ≥ 10 then + -- pred_ram_gib is column 8; integer GiB resolution suffices + -- for admission (the limit already carries an 8% margin). + match cols[0]!.toNat?, ((cols[8]!.splitOn ".")[0]!).toNat? with + | some k, some gib => m := m.insert k (gib + 1) + | _, _ => pure () + pure m + let boxGib : Nat ← do + let info ← IO.FS.readFile "/proc/meminfo" + match info.splitOn "\n" |>.head?.map (·.splitOn " " |>.filter (· ≠ "")) with + | some [_, kb, _] => pure ((kb.toNat?.getD 0) / (1024 * 1024)) + | _ => pure 0 + let limitGib := if boxGib == 0 then 0 else boxGib * 92 / 100 + let ramOf (k : Nat) : Nat := + (predRam.get? k).getD (if limitGib == 0 then 1 else limitGib) + IO.println s!"[prove] RAM-aware admission: box {boxGib} GiB, limit \ + {limitGib} GiB, {predRam.size} sidecar predictions" + let maxJobs := if jobs == 0 then shards.size else jobs + let mut failures : List (Nat × String) := [] + let mut queue := pending + let mut running : List (Nat × Nat × Task (Except IO.Error (Nat × Except String ProofRow))) := [] + let mut inFlightGib : Nat := 0 + repeat + -- Admit while the next shard fits (always admit into an idle box). + while h : queue ≠ [] do + let k := queue.head h + let need := ramOf k + if running.isEmpty + || (running.length < maxJobs && inFlightGib + need ≤ limitGib) then + let t ← IO.asTask (prio := .dedicated) do pure (k, ← proveOneShard k) + running := (k, need, t) :: running + inFlightGib := inFlightGib + need + queue := queue.tail + else + break + -- Wait for any in-flight prove to finish, then retire it. + let fin ← match running.map (·.2.2) with + | [] => break + | t :: rest => IO.waitAny (t :: rest) + let (k, res) ← match fin with + | .ok kr => pure kr + | .error e => + IO.eprintln s!"[prove] task crashed: {e}" + failures := (shards.size, toString e) :: failures + break + match res with + | .ok row => + recordProof row + done := done.insert k row.proofAddr + IO.println s!"[prove] shard {k}: proof {row.proofAddr} verified \ + ({done.size}/{shards.size})" + | .error e => + IO.eprintln s!"[prove] shard {k} FAILED: {e}" + failures := (k, e) :: failures + inFlightGib := inFlightGib - (running.find? (·.1 == k)).elim 0 (·.2.1) + running := running.filter (·.1 ≠ k) + if !failures.isEmpty then + IO.eprintln s!"[prove] {failures.length} shard(s) failed: \ + {failures.map (·.1)}; re-run the same command to resume" + return 1 + IO.println s!"[prove] OK: composed verdict — all {shards.size} shards \ + proven + verified + bound, disjoint cover" + return 0 + def runProveCmd (p : Cli.Parsed) : IO UInt32 := do - let keepGoing := p.hasFlag "keep-going" + let keepGoing := p.hasFlag "no-fail-fast" + if keepGoing && p.hasFlag "fail-fast" then + p.printError "error: --fail-fast and --no-fail-fast are mutually exclusive" + return 1 let ixePath : Option String := (p.flag? "ixe").map (·.as! String) let claimHex : Option String := (p.flag? "claim").map (·.as! String) let names := (p.variableArgsAs! String).toList @@ -161,22 +385,22 @@ def runProveCmd (p : Cli.Parsed) : IO UInt32 := do let envHandle ← match Aiur.EnvHandle.fromIxe ixe with | .error e => IO.eprintln s!"EnvHandle.fromIxe {ixe}: {e}"; return 1 | .ok h => pure h - runShardProveNative manifest envHandle ixonEnv shards k aiurSystem compiled false + let benchJson := (p.flag? "json").map fun f => + (f.as! String, + ((p.flag? "json-name").map (·.as! String)).getD s!"shard-{k}") + runShardProveNative manifest envHandle ixonEnv shards k aiurSystem + compiled false benchJson | some ixe, some manifest, none => - -- IxVM-native all-shards prove. Same envHandle reused across - -- every shard. + -- Batched, resumable all-shards prove (one env handle + one Aiur + -- system across every shard; progress in ~/.ix/cache/shard-proofs). match (← Ix.Cli.CheckCmd.loadEnvAndShards manifest ixe) with | .error e => IO.eprintln e; return 1 | .ok (ixonEnv, shards) => let envHandle ← match Aiur.EnvHandle.fromIxe ixe with | .error e => IO.eprintln s!"EnvHandle.fromIxe {ixe}: {e}"; return 1 | .ok h => pure h - let mut rc : UInt32 := 0 - for k in [0 : shards.size] do - if (← runShardProveNative manifest envHandle ixonEnv shards k - aiurSystem compiled false) != 0 then - rc := 1 - pure rc + runShardProveAllNative manifest envHandle ixonEnv shards aiurSystem + compiled (((p.flag? "jobs").map (·.as! Nat)).getD 1) | _, _, _ => Ix.Cli.CheckCmd.forEachClaim ixePath claimHex names keepGoing "prove" false runOne @@ -188,11 +412,15 @@ def proveCmd : Cli.Cmd := `[Cli| "Generate a STARK proof for an `Ix.Claim` (mirrors `ix check`'s CLI shape)" FLAGS: - "keep-going"; "Continue past failures and report them at the end instead of halting on the first." + "fail-fast"; "Halt on the first failure (the default; flag accepted for explicitness)." + "no-fail-fast"; "Continue past failures and report them at the end instead of halting on the first." "ixe" : String; "Path to a serialized `.ixe` env. When set, the binary reads the env from disk instead of using the compiled-in Lean env." "claim" : String; "32-byte hex address of a persisted `Ix.Claim` in `~/.ix/store/`. When set, proves the persisted claim against the `--ixe` env (single proof, skips per-const iteration)." "ixes" : String; "Path to a `.ixes` shard manifest (with --ixe). With --shard K: prove shard K. Without --shard: prove every shard in the partition." "shard" : Nat; "0-based shard index K (with --ixes and --ixe): prove that one shard's CheckEnv claim." + "jobs" : Nat; "Shards to prove concurrently in the all-shards batch (default 1). Each prove peaks at the shard's full predicted RAM, so raise this only when the box fits several shards at once." + json : String; "Benchmark results JSON accumulator (single-shard mode only): append a `prove-time`/`peak-rss`/`proof-size` row for the proved shard. Used by `ix bench run --backend aiur` for heavy-tier closure shards." + "json-name" : String; "Row name for --json (default: shard-)." ARGS: ...names : String; "Fully-qualified Lean.Name(s) to prove. With none, iterate every named constant in the env (sorted)." diff --git a/Ix/Cli/ShardCmd.lean b/Ix/Cli/ShardCmd.lean index 8bdee963..e5f22827 100644 --- a/Ix/Cli/ShardCmd.lean +++ b/Ix/Cli/ShardCmd.lean @@ -1,19 +1,31 @@ /- - `ix shard `: partition a profiled environment into shards, - minimizing cross-shard delta-unfold ingress (see `plans/sharding.md`). + `ix shard `: partition an environment into shards, dispatched on + the input type. - Two modes (precedence in `runShardCmd`): + `.ixe` input — the default, Aiur mode: **measured scan-and-cut**. The + env's check schedule executes through the codegen'd Aiur kernel as + thin-frontier `CheckEnv` claims with a running FFT readout, and shard + boundaries are cut where the measured cost reaches the `--max-ram` + budget's FFT equivalent (see `crates/ffi/src/aiur/scan.rs`). No profile + pass, no cost model — the manifest carries the MEASURED per-shard cost. + + `.ixe` input with `--backend zisk`: Zisk's planner from the env in one + command — the Rust-kernel profiling pass writes `.ixprof`, then + the guest-cost packer runs on it (the profile is kept: re-tuning the + budget is pure offline graph work on the `.ixprof`). + + `.ixprof` input — the profile-driven packer directly (Zisk): - default / `--max-ram G` / `--max-cycles C`: **bin-pack to a per-shard cycle/RAM cap** — the fewest shards that each stay under the budget, each packed as full as the dependency structure allows (no `--max-ram` ⇒ sized to detected system RAM). Not balanced: packing yields the minimal shard count. - `--shards N`: force exactly `N` **balanced** min-cut shards (manual override). - Reads the `.ixprof` produced by `ix profile` (pure offline graph work, so the - budget/`N` is cheap to re-tune without re-running the kernel). Writes a `.ixes` - manifest and prints a what-if report (per-shard cost + total cross-shard - ingress). The partitioner is self-contained — no external graph-library - dependency. + The `.ixprof` comes from `ix profile` (pure offline graph work, so the + budget/`N` is cheap to re-tune without re-running the kernel). Both modes + write a `.ixes` manifest (format v2, per-shard tagged costs) and print a + what-if report. The partitioner is self-contained — no external + graph-library dependency. `ix shard extract --consts `: the pipeline's scoping step — extract the named constants' dependency closure from a serialized @@ -25,8 +37,15 @@ -/ module public import Cli +public import Ix.Aiur.Compiler +public import Ix.Aiur.Protocol +public import Ix.Benchmark.Results +public import Ix.IxVM +public import Ix.IxVM.Toplevel public import Ix.KernelCheck +public import Ix.TracingTexray public import Ix.Cli.ConstsFile +public import Ix.Cli.NameOfCmd public section @@ -34,6 +53,117 @@ open Ix.KernelCheck namespace Ix.Cli.ShardCmd +/-- Shard a `.ixe` env by MEASURED cost (the default `ix shard` mode for + Aiur): execute the check schedule through the codegen'd Aiur kernel + with a running FFT readout and cut boundaries at the RAM budget's FFT + equivalent. No profile, no prediction; the manifest carries measured + per-shard cost. -/ +def runShardScan (p : Cli.Parsed) (ixePath : String) : IO UInt32 := do + let outPath : String := + match p.flag? "out" with + | some flag => flag.as! String + | none => + let base := if ixePath.endsWith ".ixe" then (ixePath.dropEnd 4).toString else ixePath + base ++ ".ixes" + let budget := (p.flag? "max-ram").map (·.as! Nat) |>.getD 250 + let eps := (p.flag? "eps").map (·.as! Nat) |>.getD 2 + let workers := (p.flag? "workers").map (·.as! Nat) |>.getD 0 + let noFailFast := p.hasFlag "no-fail-fast" + if noFailFast && p.hasFlag "fail-fast" then + p.printError "error: --fail-fast and --no-fail-fast are mutually exclusive" + return 1 + -- Deferred ranges (regions whose opening cone exceeds a fleet slot — + -- cone-bound kernel execution measured at hours of worker time on + -- FLT/Mathlib-class content) are named infeasible instead of walked + -- under fat caps, and the exclusion inventory is resolved to Lean + -- names in `.failed-names.txt`. Implies --no-fail-fast; the + -- mode reaches the scanner as fail-fast mode string "2". + let deferInfeasible := p.hasFlag "defer-infeasible" + let toplevel ← match IxVM.ixVM with + | .error e => IO.eprintln s!"Toplevel merging failed: {e}"; return 1 + | .ok t => pure t + let compiled ← match toplevel.compile with + | .error e => IO.eprintln s!"Compilation failed: {e}"; return 1 + | .ok c => pure c + let funIdx ← match compiled.getFuncIdx `verify_claim with + | some i => pure i + | none => IO.eprintln "error: verify_claim missing"; return 1 + let envHandle ← match Aiur.EnvHandle.fromIxe ixePath with + | .error e => IO.eprintln s!"EnvHandle.fromIxe {ixePath}: {e}"; return 1 + | .ok h => pure h + let workersDesc := if workers == 0 then "auto" else toString workers + IO.println s!"Scanning {ixePath} @ {budget} GiB (ε {eps}%, {workersDesc} workers)" + (← IO.getStdout).flush + -- `benchJson = (out, rowName)` reports the scan as a benchmark row + -- (the `aiur` bench backend's whole-env execute row): the scan IS a + -- whole-env execution — the cut on top of it is an in-memory merge + -- over the collected block records — so the wall reports as + -- `execute-time`, windowing the scan itself: the env mmap and + -- toplevel build are excluded, so the measure tracks the kernel + -- execution, not the loader. `peak-rss` is the process tree's + -- absolute high-water, matching the check rows' semantics. + let benchJson := (p.flag? "json").map fun f => + (f.as! String, ((p.flag? "json-name").map (·.as! String)).getD "scan") + if benchJson.isSome then + TracingTexray.startSampler + TracingTexray.resetPeakTreeRss + let start ← IO.monoMsNow + -- The compiled system feeds the scanner's analytic peak-prove-RAM + -- model (circuit widths, lookup shapes, quotient degrees); its one-time + -- build cost (preprocessed gadget commit) is seconds against a + -- minutes-scale scan. + let system := Aiur.AiurSystem.build compiled.bytecode + Aiur.defaultCommitmentParameters Aiur.defaultFriParameters + -- Process-pool mode: the scan spawns ` shard-worker` + -- children under cgroup memory caps, so an over-cap worker is + -- OOM-killed alone and recovered, never the box. + let workerBin := (← IO.appPath).toString + match Aiur.AiurSystem.scanShardsWithEnv system funIdx + envHandle (toString budget) (toString eps) (toString workers) + (if deferInfeasible then "2" + else if noFailFast then "0" + else "1") + outPath workerBin ixePath with + | .ok () => + if let some (out, rowName) := benchJson then + let secs := ((← IO.monoMsNow) - start).toFloat / 1000.0 + let peakRss ← TracingTexray.peakTreeRssBytes + Ix.Benchmark.Results.writeRow out rowName "ok" + [ ("execute-time", Ix.Benchmark.Results.jsonRound 3 secs) + , ("peak-rss", Lean.toJson peakRss) ] + IO.println s!"[shard scan] wrote {outPath} (+ .costs.csv, measured)" + -- Resolve the exclusion inventory to Lean names: one env decode + + -- batch reverse-index lookup (`ix name-of --addrs-file` semantics), + -- so a shard run over dense content ends with a readable list of + -- exactly which constants were excluded and why. + if deferInfeasible then + let failedPath := outPath ++ ".failed.csv" + if ← System.FilePath.pathExists failedPath then + let mut addrs : Array Address := #[] + for line in (← IO.FS.readFile failedPath).splitOn "\n" do + let addrStr := (line.splitOn ",").headD "" + if let some a := Address.fromString addrStr then + addrs := addrs.push a + if !addrs.isEmpty then + let bytes ← IO.FS.readBinFile ixePath + match Ixon.deEnvAnon bytes with + | .error e => + IO.eprintln s!"[shard scan] failed-names resolution skipped: {e}" + | .ok ixonEnv => + let resolved := + Ix.Cli.NameOfCmd.resolveAddrs ixonEnv addrs + let namesPath := outPath ++ ".failed-names.txt" + let lines := resolved.map fun (a, disp) => s!"{a} {disp}" + IO.FS.writeFile namesPath + (String.intercalate "\n" lines.toList ++ "\n") + IO.println + s!"[shard scan] {addrs.size} excluded block(s) resolved → \ + {namesPath}" + return 0 + | .error e => + IO.eprintln s!"error: shard scan failed: {e}" + return 1 + def runShardExtractCmd (p : Cli.Parsed) : IO UInt32 := do let some pathArg := p.positionalArg? "path" | p.printError "error: must specify to a .ixe file" @@ -73,9 +203,28 @@ def shardExtractCmd : Cli.Cmd := `[Cli| def runShardCmd (p : Cli.Parsed) : IO UInt32 := do let some pathArg := p.positionalArg? "path" - | p.printError "error: must specify to a .ixprof file" + | p.printError "error: must specify to a .ixe (measured scan) or .ixprof (profile packer)" + return 1 + let path := pathArg.as! String + -- Dispatch on the input: a `.ixe` env runs the backend's own planner + -- from the env itself — aiur (default) is the measured scan-and-cut; + -- zisk chains the Rust-kernel profiling pass into the guest-cost + -- packer, leaving the `.ixprof` next to the env so the budget can be + -- re-tuned offline without re-running the kernel. A `.ixprof` input + -- skips straight to the profile-driven packer. + let mut espPath := path + if path.endsWith ".ixe" then + match (p.flag? "backend").map (·.as! String) |>.getD "aiur" with + | "aiur" => return ← runShardScan p path + | "zisk" => + let prof := (path.dropEnd 4).toString ++ ".ixprof" + IO.println s!"Profiling {path} → {prof} (Rust-kernel pass, zisk counters)" + (← IO.getStdout).flush + rsProfileAnonFFI path prof true true "0" "zisk" + espPath := prof + | other => + p.printError s!"error: --backend must be aiur or zisk (got {other})" return 1 - let espPath := pathArg.as! String let balancePct : Nat := match p.flag? "balance" with | some flag => flag.as! Nat @@ -97,6 +246,15 @@ def runShardCmd (p : Cli.Parsed) : IO UInt32 := do match p.flag? "parallelism" with | some flag => max 1 (flag.as! Nat) | none => 1 + let backend := (p.flag? "backend").map (·.as! String) |>.getD "zisk" + if backend != "zisk" && backend != "aiur" then + p.printError s!"error: --backend must be zisk or aiur (got {backend})" + return 1 + -- Reaching here with `aiur` means a `.ixprof` input (a `.ixe` dispatched + -- to the scan above): the model packer that served it is gone. + if backend == "aiur" then + p.printError "error: the Aiur model packer was removed; run the measured scan on the .ixe instead" + return 1 -- Precedence: explicit --shards (fixed count) > explicit --max-cycles/--max-ram -- (budget) > default (size to detected system RAM). @@ -107,35 +265,91 @@ def runShardCmd (p : Cli.Parsed) : IO UInt32 := do outPath | none => if maxCycles.isNone && maxRam.isNone then - IO.println s!"Sharding {espPath} to detected system RAM (balance ±{balancePct}%)" + IO.println s!"Sharding {espPath} to detected system RAM ({backend} model, balance ±{balancePct}%)" else - IO.println s!"Sharding {espPath} to budget (max-cycles={maxCycles.getD 0}, max-ram={maxRam.getD 0} GiB, balance ±{balancePct}%)" + IO.println s!"Sharding {espPath} to budget ({backend} model, max-cycles={maxCycles.getD 0}, max-ram={maxRam.getD 0} GiB, balance ±{balancePct}%)" rsShardEspCapFFI espPath (toString (maxCycles.getD 0)) (toString (maxRam.getD 0)) - (toString balancePct) (toString parallelism) outPath + (toString balancePct) (toString parallelism) outPath backend if !outPath.isEmpty then IO.println s!"[shard] wrote {outPath}" return 0 +/-- The child side of the scan's process pool: build the same + toplevel/system/env as the parent, then hand off to the Rust worker + loop (stdin commands, stdout replies) until EOF. -/ +def runShardWorkerCmd (p : Cli.Parsed) : IO UInt32 := do + let some ixe := (p.flag? "ixe").map (·.as! String) | do + p.printError "error: shard-worker requires --ixe" + return 1 + let cutGib := (p.flag? "cut-gib").map (·.as! String) |>.getD "inf" + let batch := (p.flag? "batch").map (·.as! Nat) |>.getD 128 + let softCap := (p.flag? "soft-cap-gib").map (·.as! String) |>.getD "inf" + let pieces := (p.flag? "pieces").map (·.as! Nat) |>.getD 16 + let toplevel ← match IxVM.ixVM with + | .error e => IO.eprintln s!"Toplevel merging failed: {e}"; return 1 + | .ok t => pure t + let compiled ← match toplevel.compile with + | .error e => IO.eprintln s!"Compilation failed: {e}"; return 1 + | .ok c => pure c + let funIdx ← match compiled.getFuncIdx `verify_claim with + | some i => pure i + | none => IO.eprintln "error: verify_claim missing"; return 1 + let system := Aiur.AiurSystem.build compiled.bytecode + Aiur.defaultCommitmentParameters Aiur.defaultFriParameters + let envHandle ← match Aiur.EnvHandle.fromIxe ixe with + | .error e => IO.eprintln s!"EnvHandle.fromIxe {ixe}: {e}"; return 1 + | .ok h => pure h + match Aiur.AiurSystem.scanWorker system funIdx envHandle cutGib + (toString batch) softCap (toString pieces) + (if p.hasFlag "defer-growth" then "2" + else if p.hasFlag "exec-only" then "1" + else "0") with + | .ok () => return 0 + | .error e => IO.eprintln s!"shard-worker: {e}"; return 1 + end Ix.Cli.ShardCmd open Ix.Cli.ShardCmd in def shardCmd : Cli.Cmd := `[Cli| "shard" VIA runShardCmd; - "Partition a `.ixprof` into shards: pack to a RAM/cycle cap (default) or N balanced shards" + "Partition an env into shards. A `.ixe` input runs the MEASURED Aiur scan-and-cut (default; no profile pass); a `.ixprof` input runs the profile-driven packer (Zisk)" FLAGS: - shards : Nat; "Fixed number of shards N (overrides the default budget sizing)" - "max-cycles" : Nat; "Per-shard guest-cycle budget (overrides the default RAM sizing)" - "max-ram" : Nat; "Per-shard host-RAM budget, GiB (default: detected system RAM)" - balance : Nat; "Per-bisection balance tolerance, percent (default 5)" - parallelism : Nat; "Provers assumed for the prove-time estimate (default 1 = sequential)" - out : String; "Output .ixes manifest path (default: .ixes, e.g. init.ixprof → init.ixes)" + "max-ram" : Nat; "Per-shard host-RAM budget, GiB (scan default 250; .ixprof default: detected system RAM)" + backend : String; "Planner: aiur (default on `.ixe`: measured scan) or zisk (`.ixe`: profile pass + guest-cost pack in one command; `.ixprof`: pack directly)." + out : String; "Output .ixes manifest path (default: input basename + .ixes)" + eps : Nat; "Scan only: pre-charged cut headroom, percent (default 2): covers the batched claim readout's measured drift (~1%) plus merge-sum conservatism" + workers : Nat; "Scan only: parallel chunk scanners (default 0 = autoscale to cores and detected RAM). Each holds one segment's query record and faulted witness, so workers × segment footprint must fit the box" + "fail-fast"; "Scan: halt on the first kernel-rejected block (the default; flag accepted for explicitness)." + "no-fail-fast"; "Scan: skip kernel-rejected blocks (named as skipped, excluded from the partition, listed in .failed.csv). The manifest then does not cover them — the downstream coverage gate reports exactly which." + "defer-infeasible"; "Scan: name deferred dense regions (opening cone exceeds a fleet slot) resource-infeasible instead of walking them under fat caps, and resolve the whole exclusion inventory to Lean names in .failed-names.txt. Implies --no-fail-fast. The partition covers only tractable content; failed.csv + failed-names.txt carry the exact boundary." + json : String; "Scan only: benchmark results JSON accumulator — append an `execute-time`/`peak-rss` row (the scan wall is a whole-env execution wall). Used by `ix bench run --backend aiur` for the whole-env execute row." + "json-name" : String; "Row name for --json (default: scan)." + shards : Nat; ".ixprof only: fixed number of shards N (overrides the budget sizing)" + "max-cycles" : Nat; ".ixprof only: per-shard guest-cycle budget (overrides the RAM sizing)" + balance : Nat; ".ixprof only: per-bisection balance tolerance, percent (default 5)" + parallelism : Nat; ".ixprof only: provers assumed for the prove-time estimate (default 1 = sequential)" ARGS: - path : String; "Path to a .ixprof produced by `ix profile`" + path : String; "A serialized `.ixe` env (measured scan) or a `.ixprof` from `ix profile` (profile packer)" SUBCOMMANDS: shardExtractCmd ] +open Ix.Cli.ShardCmd in +def shardWorkerCmd : Cli.Cmd := `[Cli| + "shard-worker" VIA runShardWorkerCmd; + "INTERNAL: scan-worker child for the shard scanner's process pool — spawned automatically under a cgroup memory cap; not for direct use" + + FLAGS: + ixe : String; "Path to the `.ixe` env (same file as the parent scan)" + "cut-gib" : String; "Segment cut, GiB of predicted prove RSS (decimal string)" + batch : Nat; "Blocks per measurement claim" + "soft-cap-gib" : String; "Graceful record ceiling, GiB — segments cut here so only mid-claim growth reaches the cgroup kill" + pieces : Nat; "Schedule piece count (must match the parent's chunking)" + "exec-only"; "Execute-only mode: record-bytes cut, no prove model" + "defer-growth"; "Execute-only phase 1: hand dense range remainders back (DEFER) when measured record growth per block crosses the phase threshold; implies --exec-only" +] + end diff --git a/Ix/KernelCheck.lean b/Ix/KernelCheck.lean index 5449c0a5..fecb774c 100644 --- a/Ix/KernelCheck.lean +++ b/Ix/KernelCheck.lean @@ -198,13 +198,34 @@ opaque rsEnvExtractFFI : per-block heartbeats + the delta-unfold graph (the sharding cost model, see `plans/sharding.md`). Runs the anon kernel over every checkable target. `isolate` clears the kernel's reduction-memo caches between constants for - sound/faithful recording; `quiet` suppresses per-constant progress. -/ + sound/faithful recording; `quiet` suppresses per-constant progress. `top` + (a decimal string, kept ABI-simple) sizes the summary's per-metric block + leaderboards; "0" disables them. `backend` selects which cost models the + summary prints: "aiur", "zisk", or "all". -/ @[extern "rs_kernel_profile_anon"] opaque rsProfileAnonFFI : @& String → -- .ixe path @& String → -- .ixprof output path @& Bool → -- isolate caches @& Bool → -- quiet + @& String → -- leaderboard size (decimal) + @& String → -- backend cost models (all|aiur|zisk) + IO Unit + +/-- FFI: env-wide closure cost sweep. For every named constant (one + representative name per home block), walks its full reference closure, + sums the whole-env `.ixprof` per-block counters over the members, and + applies the Aiur execute/prove cost models. Writes a per-root CSV and + prints feasibility / min-root-per-hot-block / diversity reports to + stderr. Numeric params are decimal strings (ABI-simple). -/ +@[extern "rs_profile_sweep"] +opaque rsProfileSweepFFI : + @& String → -- .ixe path + @& String → -- .ixprof path + @& String → -- output CSV path + @& String → -- RAM budget GiB (decimal) + @& String → -- hot blocks tracked (decimal) + @& String → -- diversity representatives (decimal) IO Unit /-- FFI: partition a `.ixprof` into `numShards` shards, writing a `.ixes` @@ -222,7 +243,9 @@ opaque rsShardEspFFI : /-- FFI: partition a `.ixprof` to a per-shard cycle/RAM budget, writing a `.ixes` manifest. `maxCycles` is a guest-STEP cap; if `ramGb` > 0 it is converted via the measured prover RAM model and overrides `maxCycles`. Pass - "0" for whichever is unused. Decimal strings (ABI-simple). -/ + "0" for whichever is unused. Decimal strings (ABI-simple). `backend` + must be "zisk" (guest-STEP cap); "aiur" is rejected — Aiur shards via + the measured scan on the `.ixe`. -/ @[extern "rs_shard_esp_cap"] opaque rsShardEspCapFFI : @& String → -- .ixprof path @@ -231,6 +254,7 @@ opaque rsShardEspCapFFI : @& String → -- balance percent @& String → -- parallelism (provers for prove-time est) @& String → -- .ixes output path ("" = skip) + @& String → -- backend cost model (zisk|aiur) IO Unit end Ix.KernelCheck diff --git a/Ix/Store.lean b/Ix/Store.lean index 08c323be..5f37dc67 100644 --- a/Ix/Store.lean +++ b/Ix/Store.lean @@ -38,6 +38,18 @@ def storeDir : StoreIO FilePath := do IO.toEIO .ioError (IO.FS.createDirAll path) return path +/-- `~/.ix/cache/` — keyed, wipeable derived state, distinct + from the content-addressed store: filenames are lookup keys (claim + digests), contents are re-derivable or re-verifiable, and deleting + the directory only costs recomputation. Used for the shard-prove + resume ledger (`shard-proofs`). -/ +def cacheDir (namespace' : String) : StoreIO FilePath := do + let home ← getHomeDir + let path := home / ".ix" / "cache" / namespace' + if !(<- path.pathExists) then + IO.toEIO .ioError (IO.FS.createDirAll path) + return path + def storePath (addr: Address): StoreIO FilePath := do let store <- storeDir let hex := hexOfBytes addr.hash diff --git a/Main.lean b/Main.lean index af1bd264..a7e76ec1 100644 --- a/Main.lean +++ b/Main.lean @@ -48,6 +48,7 @@ def ixCmd : Cli.Cmd := `[Cli| profileCmd; proveCmd; shardCmd; + shardWorkerCmd; codegenCmd; verifyCmd; addrOfCmd; diff --git a/Tests/Ix/Kernel/ShardPipeline.lean b/Tests/Ix/Kernel/ShardPipeline.lean new file mode 100644 index 00000000..71131de6 --- /dev/null +++ b/Tests/Ix/Kernel/ShardPipeline.lean @@ -0,0 +1,136 @@ +/- + End-to-end regression for the Aiur shard pipeline: + + Lean env → .ixe → `ix profile` (touch graph) → `ix shard` + (measured packing) → check every shard → batch prove + (prove → verify → bind) → composed verdict → resume. + + The suite pins: + + - the packer produces a disjoint cover over a fresh touch-graph + profile; + - every shard CHECKS through the native kernel; + - the batched prove reaches the composed verdict: every proof + VERIFIES and binds to its shard's reconstructed claim; + - the shard-proofs cache makes the run resumable (second run: 0 + pending), exercising the entry re-verification path. + + Artifacts live under `.lake/tmp-shard-pipeline/` (wiped per run), + including a scratch cache root so the test never touches the global + `~/.ix/cache`; proofs land in the content-addressed store like any + prove. +-/ +import Ix.Meta +import Ix.Aiur.Protocol +import Ix.IxVM +import Ix.IxVM.ClaimHarness +import Ix.Ixon +import Ix.KernelCheck +import Ix.Cli.CheckCmd +import Ix.Cli.ProveCmd +import Ix.Cli.VerifyCmd +import LSpec + +open LSpec + +namespace Tests.Ix.Kernel.ShardPipeline + +/-- Small but non-trivial roots: enough closure (~200 consts) to pack + several shards at a small budget, with inductives, recursors, and + string/Nat literals represented. -/ +private def roots : Array Lean.Name := + #[`Nat.add, `List.append, `Nat.mul, `Char.ofNat] + +private def dir : System.FilePath := ".lake" / "tmp-shard-pipeline" + +def shardPipelineTests (env : Lean.Environment) + (compiled : Aiur.CompiledToplevel) : IO TestSeq := do + IO.FS.createDirAll dir + let ixe := dir / "pipeline.ixe" + let prof := dir / "pipeline.ixprof" + let ixes := dir / "pipeline.ixes" + let cacheRoot := dir / "cache" + for f in [ixe, prof, ixes] do + if ← f.pathExists then IO.FS.removeFile f + if ← cacheRoot.pathExists then IO.FS.removeDirAll cacheRoot + + -- Lean env → .ixe + let ixonEnv ← IxVM.ClaimHarness.loadSharedIxonEnv roots env + let bytes ← IO.ofExcept (Ixon.serEnv ixonEnv) + IO.FS.writeBinFile ixe bytes + + -- .ixe → touch-graph profile → fixed 3-shard min-cut manifest. A fixed + -- count (not the Aiur RAM packer, whose composed base exceeds any budget + -- a ~200-constant env could fill) keeps the partition multi-shard and + -- small; the claim layer is partition-agnostic, and the RAM packer has + -- its own unit coverage. + Ix.KernelCheck.rsProfileAnonFFI ixe.toString prof.toString true true "0" "aiur" + Ix.KernelCheck.rsShardEspFFI prof.toString "3" "10" "1" ixes.toString + + let (parsedEnv, shards) ← + match ← Ix.Cli.CheckCmd.loadEnvAndShards ixes.toString ixe.toString with + | .error e => throw <| IO.userError s!"manifest/env load failed: {e}" + | .ok r => pure r + let mut tests : TestSeq := + test s!"packer produced a multi-shard partition ({shards.size})" + (shards.size ≥ 2) + + -- Every shard checks through the native kernel. + let checkRc ← Ix.Cli.CheckCmd.runShardManifestAllNative ixes.toString + ixe.toString none compiled false none false + tests := tests ++ test "all shards check" (checkRc == 0) + + -- Batched prove: prove → verify → bind → compose. + let (aiurSystem, compiled') ← match ← Ix.Cli.VerifyCmd.buildBackend with + | .error e => throw <| IO.userError s!"backend build failed: {e}" + | .ok b => pure b + let envHandle ← match Aiur.EnvHandle.fromIxe ixe.toString with + | .error e => throw <| IO.userError s!"EnvHandle.fromIxe: {e}" + | .ok h => pure h + + -- Scan-and-cut over the same env: measured thin-frontier segments → + -- merge/re-measure → manifest. The budget barely clears the RAM-model + -- base, so the tiny env still exercises cuts, the merge pass, and the + -- refine rounds; the scanned partition must cover and check like the + -- profiled one. + let scanIxes := dir / "pipeline-scan.ixes" + let scanFunIdx ← match compiled.getFuncIdx `verify_claim with + | some i => pure i + | none => throw <| IO.userError "verify_claim missing" + let scanSystem := Aiur.AiurSystem.build compiled.bytecode + Aiur.productionCommitmentParameters Aiur.productionFriParameters + -- Empty worker-bin/ixe strings select the in-process thread pool: the + -- test binary cannot exec itself as `ix shard-worker`. + match Aiur.AiurSystem.scanShardsWithEnv scanSystem scanFunIdx + envHandle "20" "5" "2" "1" scanIxes.toString "" "" with + | .error e => throw <| IO.userError s!"shard scan failed: {e}" + | .ok () => pure () + let (_, scanShards) ← + match ← Ix.Cli.CheckCmd.loadEnvAndShards scanIxes.toString ixe.toString with + | .error e => throw <| IO.userError s!"scan manifest load failed: {e}" + | .ok r => pure r + tests := tests ++ + test s!"scan produced a covering partition ({scanShards.size} shard(s))" + (scanShards.size ≥ 1) + let scanCheckRc ← Ix.Cli.CheckCmd.runShardManifestAllNative scanIxes.toString + ixe.toString none compiled false none false + tests := tests ++ test "all scanned shards check" (scanCheckRc == 0) + let proveRc ← Ix.Cli.ProveCmd.runShardProveAllNative ixes.toString envHandle + parsedEnv shards aiurSystem compiled' 1 (some cacheRoot) + tests := tests ++ test "batch prove reaches the composed verdict" (proveRc == 0) + let cacheEntries (ns : String) : IO Nat := do + if !(← (cacheRoot / ns).pathExists) then return 0 + return (← (cacheRoot / ns).readDir).size + tests := tests ++ test + s!"shard-proofs cache written ({← cacheEntries "shard-proofs"} entries)" + ((← cacheEntries "shard-proofs") == shards.size) + + -- Resume: every cached proof must re-verify and be skipped; verdict + -- unchanged. + let resumeRc ← Ix.Cli.ProveCmd.runShardProveAllNative ixes.toString envHandle + parsedEnv shards aiurSystem compiled' 1 (some cacheRoot) + tests := tests ++ test "resume re-verifies cached proofs and stays green" + (resumeRc == 0) + pure tests + +end Tests.Ix.Kernel.ShardPipeline diff --git a/Tests/Main.lean b/Tests/Main.lean index 3eab7ece..17ec330f 100644 --- a/Tests/Main.lean +++ b/Tests/Main.lean @@ -26,6 +26,7 @@ import Tests.Ix.Kernel.Roundtrip import Tests.Ix.Kernel.RoundtripNoCompile import Tests.Ix.Kernel.Tutorial import Tests.Ix.Kernel.Arena +import Tests.Ix.Kernel.ShardPipeline import Tests.Ix.Kernel.PrimAddrs import Tests.Ix.RustSerialize import Tests.Ix.RustDecompile @@ -225,9 +226,14 @@ def ignoredRunners (env : Lean.Environment) : List (String × IO UInt32) := [ pure (LSpec.test s!"Shard pipeline FFT matches: expected 10817625733, got {actual}" (actual = 10_785_479_733)) + -- Full planner-to-composed-verdict E2E: profile → pack → check + -- every shard → batched prove with resume. + let shardPipeSeq ← + Tests.Ix.Kernel.ShardPipeline.shardPipelineTests env v2Env.compiled LSpec.lspecIO (.ofList [("ixvm", - [fullSeq, aiurSeq, arenaSeq, exploitSeq, paritySeq, shardSeq])]) []), + [fullSeq, aiurSeq, arenaSeq, exploitSeq, paritySeq, shardSeq, + shardPipeSeq])]) []), ("validate-aux", runCompileValidateAux env), -- Cross-compiler differential over the same fixture corpus: pure-Lean -- Ix.CompileM per-block vs Rust, root-cause classified (see diff --git a/crates/aiur/src/execute.rs b/crates/aiur/src/execute.rs index 69e03e12..68ea1c30 100644 --- a/crates/aiur/src/execute.rs +++ b/crates/aiur/src/execute.rs @@ -38,27 +38,107 @@ impl QueryRecord { } } +#[derive(Clone, Copy)] pub struct IOKeyInfo { pub idx: usize, pub len: usize, } +/// On-demand witness supplier for an [`IOBuffer`]: when execution asks for +/// a `(channel, key)` the buffer does not hold, the backing gets one chance +/// to produce the data, which is then materialized into the buffer exactly +/// as if the host had seeded it eagerly. Execution cannot observe the +/// difference — same bytes, same keys, same in-circuit verification — so +/// host witness RAM scales with the FAULTED set instead of the shipped +/// closure. `Send + Sync` because scan/check workers share one source. +pub trait IOFaultSource: Send + Sync { + fn fault(&self, channel: G, key: &[G]) -> Option>; +} + pub struct IOBuffer { /// Per-channel data arenas. `idx` slots into `data[&channel]`. pub data: FxHashMap>, /// Channel-keyed info map; same `key` on different channels resolves /// to distinct `IOKeyInfo`. pub map: FxHashMap<(G, Vec), IOKeyInfo>, + /// Lazy witness backing; `None` means fully host-seeded (the eager + /// builders and the Lean-marshalled buffers). + pub backing: Option>, +} + +impl Default for IOBuffer { + fn default() -> Self { + Self::new() + } } impl IOBuffer { + pub fn new() -> Self { + Self { + data: FxHashMap::default(), + map: FxHashMap::default(), + backing: None, + } + } + + pub fn with_backing(backing: std::sync::Arc) -> Self { + Self { + data: FxHashMap::default(), + map: FxHashMap::default(), + backing: Some(backing), + } + } + + fn invalid_key(channel: G, key: &[G]) -> ExecError { + // Name the channel and key: a missing witness entry is otherwise + // indistinguishable from any other, and the key identifies the + // constant or blob whose bytes the host failed to seed. + let hex: String = key + .iter() + .map(|g| format!("{:02x}", g.as_canonical_u64() & 0xff)) + .collect(); + ExecError::InvalidIOKey { channel: channel.as_canonical_u64(), key: hex } + } + + /// Execution-path lookup: on a miss, the `backing` (if any) may + /// materialize the entry into the buffer before the lookup fails. + /// `&mut` because a fault appends to the arena; both the interpreter + /// and the codegen'd kernels thread `&mut IOBuffer` already. #[inline] pub fn get_info( + &mut self, + channel: G, + key: &[G], + ) -> Result { + if let Some(info) = self.map.get(&(channel, key.to_vec())) { + return Ok(*info); + } + if let Some(src) = &self.backing + && let Some(data) = src.fault(channel, key) + { + let arena = self.data.entry(channel).or_default(); + let info = IOKeyInfo { idx: arena.len(), len: data.len() }; + arena.extend(data); + self.map.insert((channel, key.to_vec()), info); + return Ok(info); + } + Err(Self::invalid_key(channel, key)) + } + + /// Read-only lookup for post-execution passes (trace generation runs + /// circuits in parallel over a shared buffer): never faults — by trace + /// time execution has materialized every entry it read. + #[inline] + pub fn get_info_frozen( &self, channel: G, key: &[G], - ) -> Result<&IOKeyInfo, ExecError> { - self.map.get(&(channel, key.to_vec())).ok_or(ExecError::InvalidIOKey) + ) -> Result { + self + .map + .get(&(channel, key.to_vec())) + .copied() + .ok_or_else(|| Self::invalid_key(channel, key)) } fn set_info( &mut self, @@ -123,7 +203,10 @@ pub enum ExecError { MatchNoCase(u64), NoContinuation, StackNotEmpty, - InvalidIOKey, + InvalidIOKey { + channel: u64, + key: String, + }, IOMappingAlreadySet, IOReadOutOfBounds { idx: usize, @@ -165,7 +248,9 @@ impl std::fmt::Display for ExecError { Self::StackNotEmpty => { write!(f, "exec entries stack not empty at return") }, - Self::InvalidIOKey => write!(f, "invalid IO key"), + Self::InvalidIOKey { channel, key } => { + write!(f, "invalid IO key: channel {channel}, key {key}") + }, Self::IOMappingAlreadySet => write!(f, "IO mapping already set for key"), Self::IOReadOutOfBounds { idx, len } => { write!(f, "IO read out of bounds: idx={idx}, len={len}") @@ -186,7 +271,13 @@ static QUERY_STATS: std::sync::LazyLock = std::env::var_os("IX_AIUR_QUERY_STATS").is_some() }); -fn dump_query_stats(record: &QueryRecord, tag: &str) { +/// Whether `IX_AIUR_QUERY_STATS=1` is set (the codegen'd execution paths +/// share `QueryRecord`, so callers there gate their own dumps on this). +pub fn query_stats_enabled() -> bool { + *QUERY_STATS +} + +pub fn dump_query_stats(record: &QueryRecord, tag: &str) { let mut rows: Vec<(usize, usize, usize)> = record .function_queries .iter() @@ -219,18 +310,110 @@ impl Toplevel { args: Vec, io_buffer: &mut IOBuffer, ) -> Result<(QueryRecord, Vec), ExecError> { + let mut record = QueryRecord::new(self); + let output = + self.execute_with_record(fun_idx, args, io_buffer, &mut record)?; + Ok((record, output)) + } + + /// Like [`Self::execute`] but accumulating into a caller-owned + /// [`QueryRecord`]: repeated calls share the memo tables, so a query + /// resolved by an earlier call is a hit rather than re-executed. This is + /// how a sequence of independent claims is executed with the same + /// memoization semantics as one combined run (the scan-and-cut sharder's + /// segment loop). + pub fn execute_with_record( + &self, + fun_idx: FunIdx, + args: Vec, + io_buffer: &mut IOBuffer, + record: &mut QueryRecord, + ) -> Result, ExecError> { if !self.functions[fun_idx].entry { return Err(ExecError::NotEntryFunction(fun_idx)); } - let mut record = QueryRecord::new(self); let function = &self.functions[fun_idx]; - let output = - function.execute(fun_idx, args, self, &mut record, io_buffer)?; + let output = function.execute(fun_idx, args, self, record, io_buffer)?; if *QUERY_STATS { - dump_query_stats(&record, "final"); + dump_query_stats(record, "final"); } - Ok((record, output)) + Ok(output) + } +} + +/// Total FFT cost of a [`QueryRecord`]: `Σ width·height·log2(max(height,2))` +/// over every constrained function circuit plus the memory circuits, with +/// heights = unique memoized queries. Mirrors `Ix/Aiur/Statistics.lean`'s +/// `computeStats.totalFftCost` (function width `layout.totalWidth` = +/// `width + extDegree·max(lookups,1)`; memory width `3 + size + extDegree`; +/// gadget circuits excluded, as there), so a running readout here matches +/// the number the check stats dump prints and the RAM/wall lines are +/// calibrated against. +/// `usize → f64` without a lossy `as` cast: split into `u32` halves, each +/// exactly convertible. Matches `n as f64` bit-for-bit below 2^52 (query +/// counts and widths stay far below) and rounds identically above. +pub fn f64_from_usize(n: usize) -> f64 { + let hi = u32::try_from(n >> 32).expect("usize is at most 64 bits"); + let lo = u32::try_from(n & 0xFFFF_FFFF).expect("masked to u32 range"); + f64::from(hi) * 4_294_967_296.0 + f64::from(lo) +} + +/// Approximate resident bytes of a [`QueryRecord`]: retained key/output +/// field elements (8 B each) plus ~21 B of per-entry index overhead +/// (stored hash 8, multiplicity element 8, hash-table slot ~5 with load +/// factor). The memory-circuit stores +/// dominate on arithmetic-heavy content, where entries are FFT-cheap +/// (narrow columns) but RAM-heavy — the second resource dimension a +/// RAM-budgeted partition has to price alongside FFT cost. +pub fn record_retained_bytes(record: &QueryRecord) -> usize { + let mut elems = 0usize; + let mut entries = 0usize; + for m in &record.function_queries { + elems += m.retained_elems(); + entries += m.len(); + } + for (_, m) in &record.memory_queries { + elems += m.retained_elems(); + entries += m.len(); + } + elems * 8 + entries * 21 +} + +/// Exact resident heap of the record's query maps (arena fill at hugepage +/// granularity, stored hashes, hash-table allocations). This is the +/// scanner's record threshold metric — what the process actually holds +/// while executing. [`record_retained_bytes`] intentionally differs: it +/// is the analytic prove-RAM model's record term, calibrated end-to-end +/// against measured proves, and must not change with allocator details. +pub fn record_heap_bytes(record: &QueryRecord) -> usize { + record.function_queries.iter().map(QueryMap::heap_bytes).sum::() + + record + .memory_queries + .iter() + .map(|(_, m)| m.heap_bytes()) + .sum::() +} + +pub fn record_fft_cost(toplevel: &Toplevel, record: &QueryRecord) -> f64 { + const EXT_DEGREE: usize = 2; + fn fft(w: usize, h: usize) -> f64 { + if h == 0 { + 0.0 + } else { + f64_from_usize(w) * f64_from_usize(h) * f64_from_usize(h.max(2)).log2() + } + } + let mut total = 0.0; + for (i, f) in toplevel.functions.iter().enumerate() { + if f.constrained { + let w = f.layout.width() + EXT_DEGREE * f.layout.lookups.max(1); + total += fft(w, record.function_queries[i].len()); + } + } + for (size, qm) in &record.memory_queries { + total += fft(3 + size + EXT_DEGREE, qm.len()); } + total } enum ExecEntry<'a> { @@ -395,8 +578,8 @@ impl Function { let channel = map[*channel]; let key = key.iter().map(|v| map[*v]).collect::>(); let IOKeyInfo { idx, len } = io_buffer.get_info(channel, &key)?; - map.push(G::from_usize(*idx)); - map.push(G::from_usize(*len)); + map.push(G::from_usize(idx)); + map.push(G::from_usize(len)); }, ExecEntry::Op(Op::IOSetInfo(channel, key, idx, len)) => { let channel = map[*channel]; diff --git a/crates/aiur/src/querymap.rs b/crates/aiur/src/querymap.rs index 56f97306..81576f1d 100644 --- a/crates/aiur/src/querymap.rs +++ b/crates/aiur/src/querymap.rs @@ -194,6 +194,14 @@ impl SegStore { fn retained_elems(&self) -> usize { self.entries * self.stride } + + /// Heap bytes of the arena's fill. Segments are mmap-backed and fault + /// as they fill, so fill IS residency up to page granularity — never + /// count reserved-but-untouched capacity (a fresh segment per store + /// across hundreds of maps adds gigabytes of phantom bytes). + fn heap_bytes(&self) -> usize { + self.entries * self.stride * size_of::() + } } /// Segmented store of per-entry key hashes. Kept so hash-table growth can @@ -210,6 +218,19 @@ impl SegHashes { Self { segs: Vec::new(), entries: 0 } } + /// Visit every stored hash in insertion order. One sequential pass + /// over the segment arenas — no rehashing, no table walk. + fn for_each(&self, mut f: impl FnMut(u64)) { + let mut left = self.entries; + for seg in &self.segs { + let take = left.min(seg.len); + for &h in seg.slice(0, take) { + f(h); + } + left -= take; + } + } + #[inline] fn at(&self, i: usize) -> u64 { self.segs[i >> SEG_BITS].slice(i & SEG_MASK, 1)[0] @@ -224,6 +245,11 @@ impl SegHashes { self.segs[seg].extend_from_slice(&[h]); self.entries += 1; } + + /// Heap bytes of the fill; same rule as [`SegStore::heap_bytes`]. + fn heap_bytes(&self) -> usize { + self.entries * size_of::() + } } /// Append-only query store with a hash index. @@ -282,6 +308,30 @@ impl QueryMap { self.keys.retained_elems() + self.outs.retained_elems() } + /// Visit the stored 64-bit hash of every unique entry, in insertion + /// order. The hashes are already computed and resident (they back + /// table growth), so this is a pure sequential read — the scanner's + /// union-pricing sketches are built from these without rehashing. + pub fn for_each_hash(&self, f: impl FnMut(u64)) { + self.hashes.for_each(f); + } + + /// Exact heap bytes of this map's fill: arena elements, stored hashes, + /// and the hash-table allocation (power-of-two buckets at 7/8 load, + /// `u32` index + 1 control byte per bucket, fully resident). + pub fn heap_bytes(&self) -> usize { + let buckets = if self.table.capacity() == 0 { + 0 + } else { + (self.table.capacity() * 8 / 7 + 1).next_power_of_two() + }; + self.keys.heap_bytes() + + self.outs.heap_bytes() + + self.mults.heap_bytes() + + self.hashes.heap_bytes() + + buckets * (size_of::() + 1) + } + pub fn get_index_of(&self, key: &[G]) -> Option { debug_assert_eq!(key.len(), self.keys.stride); let hash = hash_g_slice(key); diff --git a/crates/aiur/src/synthesis.rs b/crates/aiur/src/synthesis.rs index cf5083b0..b11b5fc6 100644 --- a/crates/aiur/src/synthesis.rs +++ b/crates/aiur/src/synthesis.rs @@ -63,7 +63,24 @@ pub struct CircuitShape { pub preprocessed_height: usize, } +/// Per-phase breakdown from [`AiurSystem::peak_prove_bytes`]; `peak` is +/// the max phase plus the preprocessed residency. +pub struct PeakProveBytes { + pub phase_witness: usize, + pub phase_stage2: usize, + pub phase_open: usize, + pub preprocessed: usize, + pub peak: usize, +} + impl AiurSystem { + /// The bytecode this system was compiled from (drivers that execute + /// through the system — the shard scanner — read it here instead of + /// re-decoding their own copy). + pub fn toplevel(&self) -> &Toplevel { + &self.toplevel + } + pub fn build( toplevel: Toplevel, commitment_parameters: CommitmentParameters, @@ -170,6 +187,191 @@ impl AiurSystem { .collect() } + /// Predicted peak prover resident bytes for a record, from circuit + /// shapes alone — the analytic counterpart of an empirical GiB-per-fft + /// line. Mirrors this system's actual allocation schedule (multi-stark + /// rev `be1755e`; the shard-pipeline E2E asserts prediction against a + /// measured prove, so a schedule change upstream turns a test red + /// instead of silently skewing every shard-RAM prediction): + /// + /// 1. WITNESS phase: the `QueryRecord` plus every circuit's padded main + /// trace and base-field lookup witness, built in parallel and all + /// alive at once (`prove`/`prove_ixvm` drop the record only after). + /// 2. STAGE-2 transition: stage-1 LDEs and their Merkle tree, the + /// still-alive lookup witness, the logUp message array plus its + /// batch-inverse copy, and the new extension traces. + /// 3. FRI OPEN: all committed LDEs (main + stage-2 + quotient, at + /// `8·2^log_blowup` bytes per trace cell) and their trees, the + /// retained FRI fold layers (geometric in `max_log_arity`), and the + /// open-phase buffers — all proportional to `H = blowup · tallest`. + /// + /// The peak is the max of the three plus the preprocessed-gadget + /// residency committed at setup. Heights are `next_power_of_two` of the + /// record's unique queries — the padding the trace actually commits, + /// which per-fft models blur. + pub fn peak_prove_bytes(&self, record: &QueryRecord) -> PeakProveBytes { + self.peak_prove_bytes_from_raws( + &self.circuit_raws(record), + crate::execute::record_retained_bytes(record), + ) + } + + /// Per-circuit raw (unpadded) trace heights of a record, in canonical + /// system order: the record's unique queries per function/memory + /// circuit, and the byte gadgets' fixed full-table heights. + pub fn circuit_raws(&self, record: &QueryRecord) -> Vec { + self + .circuit_types() + .iter() + .map(|ct| match ct { + CircuitType::Function { idx } => record.function_queries[*idx].len(), + CircuitType::Memory { width } => { + record.memory_queries.get(width).map_or(0, |m| m.len()) + }, + CircuitType::Bytes1 => 256, + CircuitType::Bytes2 => 65536, + }) + .collect() + } + + /// [`Self::peak_prove_bytes`] from per-circuit raw heights and a record + /// byte count directly — the scanner's union pricing feeds ESTIMATED + /// union heights of a merged shard here, where no single record exists. + pub fn peak_prove_bytes_from_raws( + &self, + raws: &[usize], + record_bytes: usize, + ) -> PeakProveBytes { + const S: usize = 8; // bytes per base field element (Goldilocks) + const DG: usize = 32; // blake3 digest bytes (Merkle nodes, arity 2) + let b = 1usize << self.commitment_parameters.log_blowup; + let fold = 1usize << self.fri_parameters.max_log_arity; + let mut witness = 0usize; + let mut s1_lde = 0usize; // stage-1 LDEs + let mut lookup_w = 0usize; // base-field lookup witness + let mut msgs = 0usize; // logUp messages (+ inverse copy) + let mut s2_trace = 0usize; // stage-2 extension traces + let mut committed = 0usize; // all committed LDE bytes + let mut prep = 0usize; + let mut tallest = 0usize; + for (i, &raw) in raws.iter().enumerate().take(self.system.circuits.len()) + { + if raw == 0 { + continue; + } + let n = raw.next_power_of_two(); + tallest = tallest.max(n); + let c = &self.system.circuits[i]; + let d = c.stage_2_width / (1 + c.num_lookups); // extension degree + let args: usize = self.slot_widths[i].iter().sum(); + let q = c.quotient_degree(); + witness += + S * n * c.main_width + S * n * (c.num_lookups + args) + 40 * raw; + s1_lde += S * b * n * c.main_width; + lookup_w += S * n * (c.num_lookups + args); + msgs += 2 * S * d * n * c.num_lookups; + s2_trace += S * n * c.stage_2_width; + committed += S * b * n * (c.main_width + c.stage_2_width + q * d); + prep += S * (1 + b) * c.preprocessed_width * c.preprocessed_height + + 2 * DG * b * c.preprocessed_height; + } + let h = b * tallest; + let phase_witness = record_bytes + witness; + let phase_stage2 = s1_lde + 2 * DG * h + lookup_w + msgs + s2_trace; + // Trees (3 rounds) + retained FRI fold layers + open buffers, all ∝ H. + let fri_layers = (2 * S + 2 * DG) * h * fold / (fold - 1).max(1); + let phase_open = committed + 3 * 2 * DG * h + fri_layers + 11 * S * h; + PeakProveBytes { + phase_witness, + phase_stage2, + phase_open, + preprocessed: prep, + peak: phase_witness.max(phase_stage2).max(phase_open) + prep, + } + } + + /// Model a (possibly merged) execution from per-map unique-entry + /// estimates instead of a live record — the scanner's union pricing: + /// each element is `(is_function, id, est_raw, est_elems)` where `id` + /// is a function index or a memory width, `est_raw` the estimated + /// unique queries of the union, and `est_elems` its retained field + /// elements. Unconstrained-function maps carry no circuit and + /// contribute only record bytes, exactly as in a live record. Returns + /// the same `(fft, peak-prove-bytes)` pair a single cold record of the + /// union would produce. + pub fn model_from_map_estimates( + &self, + ests: &[(bool, usize, usize, usize)], + ) -> (f64, PeakProveBytes) { + let mut raws = vec![0usize; self.system.circuits.len()]; + let mut record_bytes = 0usize; + let types = self.circuit_types(); + for &(is_function, id, est_raw, est_elems) in ests { + record_bytes += est_elems * 8 + est_raw * 21; + let pos = types.iter().position(|ct| match ct { + CircuitType::Function { idx } => is_function && *idx == id, + CircuitType::Memory { width } => !is_function && *width == id, + _ => false, + }); + if let Some(pos) = pos { + raws[pos] = est_raw; + } + } + for (i, ct) in types.iter().enumerate() { + match ct { + CircuitType::Bytes1 => raws[i] = 256, + CircuitType::Bytes2 => raws[i] = 65536, + _ => {}, + } + } + ( + self.fft_cost_from_raws(&raws), + self.peak_prove_bytes_from_raws(&raws, record_bytes), + ) + } + + /// [`Self::fft_cost_from_raws`] of a live record. + pub fn fft_cost_of_record(&self, record: &QueryRecord) -> f64 { + self.fft_cost_from_raws(&self.circuit_raws(record)) + } + + /// FFT cost of a proof at the given per-circuit raw heights — the Rust + /// mirror of the Lean model (`Ix/Aiur/Statistics.lean`, grounded in the + /// pinned prover's actual transforms): per circuit, `(B+1)` size-`h` + /// transforms per committed column (main + stage-2 + quotient chunks), + /// plus the two quotient-rebasing transforms. Raw heights stay unpadded + /// so one-row changes remain visible. The scanner's manifest costs use + /// this, so whole-env, per-shard, and per-constant fft figures share + /// one unit. + pub fn fft_cost_from_raws(&self, raws: &[usize]) -> f64 { + fn transform(x: usize) -> f64 { + if x == 0 { + 0.0 + } else { + let xf = crate::execute::f64_from_usize(x); + xf * xf.max(2.0).log2() + } + } + let b = + crate::execute::f64_from_usize(1usize << self.commitment_parameters.log_blowup); + let mut total = 0.0f64; + for (c, &raw) in self.system.circuits.iter().zip(raws) { + if raw == 0 { + continue; + } + let d = c.stage_2_width / (1 + c.num_lookups); // extension degree + let q = c.quotient_degree(); + let q_d = q * d; + let commit_w = c.main_width + c.stage_2_width + q_d; + total += (b + 1.0) + * crate::execute::f64_from_usize(commit_w) + * transform(raw) + + crate::execute::f64_from_usize(d) * transform(q * raw) + + crate::execute::f64_from_usize(q_d) * transform(raw); + } + total + } + #[tracing::instrument(level = "info", skip_all, name = "aiur/prove")] pub fn prove( &self, @@ -263,6 +465,20 @@ impl AiurSystem { executor(&self.toplevel, fun_idx, input.to_vec(), io_buffer) .expect("IxVM-native Aiur execution failed during prove_ixvm"); drop(_g); + if std::env::var_os("IX_AIUR_PRED_RAM").is_some() { + const GIB: f64 = 1_073_741_824.0; + let p = self.peak_prove_bytes(&query_record); + let gib = |b: usize| crate::execute::f64_from_usize(b) / GIB; + eprintln!( + "[aiur] predicted peak prove RSS {:.2} GiB (witness {:.2} / \ + stage2 {:.2} / open {:.2} / preprocessed {:.2})", + gib(p.peak), + gib(p.phase_witness), + gib(p.phase_stage2), + gib(p.phase_open), + gib(p.preprocessed), + ); + } let _g = tracing::info_span!("aiur/witness").entered(); let circuit_types = self.circuit_types(); @@ -324,7 +540,6 @@ mod tests { p3_field::PrimeCharacteristicRing, types::{CommitmentParameters, FriParameters}, }; - use rustc_hash::FxHashMap; /// Small FRI parameters mirroring `vk_codec`'s test config: cheap to prove /// while still exercising the full FRI pipeline (log_blowup 1, 64 queries, @@ -342,7 +557,7 @@ mod tests { } fn empty_io_buffer() -> IOBuffer { - IOBuffer { data: FxHashMap::default(), map: FxHashMap::default() } + IOBuffer::new() } /// Hand-build the toplevel for a single constrained function `f(a, b) = a*b`. diff --git a/crates/aiur/src/trace.rs b/crates/aiur/src/trace.rs index fe107ab1..7a2aec8f 100644 --- a/crates/aiur/src/trace.rs +++ b/crates/aiur/src/trace.rs @@ -377,8 +377,8 @@ impl Op { let channel = map[*channel].0; let key = key.iter().map(|a| map[*a].0).collect::>(); let IOKeyInfo { idx, len } = - io_buffer.get_info(channel, &key).expect("Invalid IO key"); - for f in [G::from_usize(*idx), G::from_usize(*len)] { + io_buffer.get_info_frozen(channel, &key).expect("Invalid IO key"); + for f in [G::from_usize(idx), G::from_usize(len)] { map.push((f, 1)); slice.push_auxiliary(index, f); } diff --git a/crates/ffi/src/aiur.rs b/crates/ffi/src/aiur.rs index 01ad45ac..d97549b7 100644 --- a/crates/ffi/src/aiur.rs +++ b/crates/ffi/src/aiur.rs @@ -1,6 +1,7 @@ use multi_stark::p3_field::integers::QuotientMap; pub mod protocol; +pub mod scan; pub mod toplevel; use aiur::G; diff --git a/crates/ffi/src/aiur/protocol.rs b/crates/ffi/src/aiur/protocol.rs index 5e541754..3c347e5e 100644 --- a/crates/ffi/src/aiur/protocol.rs +++ b/crates/ffi/src/aiur/protocol.rs @@ -462,8 +462,9 @@ extern "C" fn rs_aiur_toplevel_check_addr_with_env( let env = &env_handle.get().env; let (_claim, input, mut io_buffer) = - match ixvm_codegen::aiur_ixvm_witness::build_claim_check_witness(env, &addr) - { + match ixvm_codegen::aiur_ixvm_witness::build_claim_check_witness_lazy( + env, &addr, + ) { Ok(t) => t, Err(e) => { return LeanExcept::error_string(&format!("witness build: {e}")); @@ -515,7 +516,7 @@ extern "C" fn rs_aiur_toplevel_shard_check_with_env( // shard entrypoint builds the the kernel witness (thin frontier + wrapper // augmentation). let (_claim, input, mut io_buffer) = - match ixvm_codegen::aiur_ixvm_witness::build_shard_check_env_witness( + match ixvm_codegen::aiur_ixvm_witness::build_shard_check_env_witness_lazy( env, &owned, ) { Ok(t) => t, @@ -566,8 +567,9 @@ extern "C" fn rs_aiur_system_prove_addr_with_env( let env = &env_handle.get().env; let (claim, input, mut io_buffer) = - match ixvm_codegen::aiur_ixvm_witness::build_claim_check_witness(env, &addr) - { + match ixvm_codegen::aiur_ixvm_witness::build_claim_check_witness_lazy( + env, &addr, + ) { Ok(t) => t, Err(e) => { return LeanExcept::error_string(&format!("witness build: {e}")); @@ -605,7 +607,7 @@ extern "C" fn rs_aiur_system_shard_prove_with_env( let env = &env_handle.get().env; let (claim, input, mut io_buffer) = - match ixvm_codegen::aiur_ixvm_witness::build_shard_check_env_witness( + match ixvm_codegen::aiur_ixvm_witness::build_shard_check_env_witness_lazy( env, &owned, ) { Ok(t) => t, @@ -763,7 +765,7 @@ fn decode_io_buffer( ) -> IOBuffer { let data = decode_io_buffer_data(io_data_arr); let map = decode_io_buffer_map(io_map_arr); - IOBuffer { data, map } + IOBuffer { data, map, backing: None } } /// Build a Lean `Array (G × Array G)` enumerating the per-channel diff --git a/crates/ffi/src/aiur/scan.rs b/crates/ffi/src/aiur/scan.rs new file mode 100644 index 00000000..c773fd36 --- /dev/null +++ b/crates/ffi/src/aiur/scan.rs @@ -0,0 +1,2652 @@ +//! Scan-and-cut sharding: shard boundaries from Aiur's own measured cost. +//! +//! Instead of predicting shard cost from profile counters, the env's check +//! schedule is EXECUTED through the codegen'd circuit kernel, and a shard +//! boundary is cut where the analytic peak-prove-RSS prediction computed +//! from the running record's circuit shapes +//! ([`AiurSystem::peak_prove_bytes`]) reaches the margined RAM budget. +//! Execution is the mandatory prefix of proving, so the measurement is the +//! prove's own cost, not a proxy — the failure mode where a recorder-side +//! counter under-represents circuit work by a content-dependent factor +//! cannot occur. +//! +//! The measurement unit is the SAME claim the prover pays for: a +//! thin-frontier `CheckEnv`, one per BATCH of schedule blocks. Each +//! segment grows batch by batch — execute the batch's claim against a +//! shared `QueryRecord`, checkpoint (fft, record bytes), continue while +//! under the cut — so every constant is checked once per segment and +//! dependencies stop at the assumed frontier. (Per-constant +//! `Check{assumptions: None}` claims are NOT usable here: without a +//! frontier the kernel checks the constant's whole dependency closure, +//! which measures 100-1000× the real per-block shard cost and rederives +//! the env spine per segment.) The shared record over-counts slightly — +//! each claim walks its own owned/assumption trees, and members assumed +//! by one claim may be checked by the next — but batching divides that +//! per-claim overhead by the batch size and removes intra-batch frontier +//! edges entirely, so the checkpoint is a tight upper bound on the +//! emitted SEGMENT's cold cost. Across segments the merge does not sum: +//! each segment ships a per-circuit membership sketch of its record, and +//! a shard is priced as the modeled cost of the UNION of its segments' +//! records — the same dedup the prove's single cold record performs — +//! so the manifest cost needs no cold re-price pass. +//! +//! Witness bytes are served LAZILY (`EnvFaultSource`): only the claim +//! wires are seeded per attempt, and constant/hint/blob bytes materialize +//! on first fault, so a worker's buffer holds one segment's touched set — +//! exactly the real shard-prove start state. +//! +//! Parallelism follows SP1's splicing design: the schedule is pre-cut into +//! coarse chunks whose edges are FORCED shard boundaries, and chunks scan +//! concurrently. A segment's start state is a cold memo table — exactly +//! the state a real shard prove begins in — so chunk-parallel scanning is +//! faithful by construction. A post-pass merges adjacent segments whose +//! FFT sum stays under the cut (the sum over-estimates the merged cost, so +//! the merged shard still fits), which decouples the chunk count — a pure +//! parallelism knob — from pack density. The schedule itself is the +//! byte-weighted min-cut linearization of the env's reference graph +//! (static, no profiling run), which keeps closure-overlapping blocks +//! adjacent so intra-segment memoization absorbs shared-dependency work. +//! +//! What crosses a boundary: nothing. Constants are order-independent +//! obligations; cross-shard soundness is the thin-frontier assumption tree +//! the claim layer already provides. + +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::{Arc, Mutex}; + +use rustc_hash::FxHashMap; + +use aiur::{ + bytecode::Toplevel, + execute::{ + IOBuffer, QueryRecord, dump_query_stats, f64_from_usize, + query_stats_enabled, record_fft_cost, record_heap_bytes, + }, + synthesis::AiurSystem, +}; +use ix_common::address::Address; +use ix_kernel::profile::{OpCounts, ProfileBuilder}; +use ix_kernel::shard::{ + ShardCost, ShardInfo, ShardManifest, aiur_prove_secs_for_fft, + balanced_agg_tree, cost_fft, cut_coherent_order, +}; +use ixon::constant::ConstantInfo as IxonCI; +use ixon::env::Env as IxonEnv; +use ixvm_codegen::aiur_ixvm_runner::execute_ixvm_with_record; +use ixvm_codegen::aiur_ixvm_witness::{ + EnvFaultSource, seed_shard_check_env_claim, +}; + +/// Bytes per GiB. +const GIB: f64 = 1_073_741_824.0; + +/// Current process resident set in GiB (`/proc/self/status` VmRSS); +/// 0 where unreadable (non-Linux). Reported in segment logs. +fn process_rss_gib() -> f64 { + let Ok(s) = std::fs::read_to_string("/proc/self/status") else { + return 0.0; + }; + let Some(rest) = s.lines().find_map(|l| l.strip_prefix("VmRSS:")) else { + return 0.0; + }; + let kib: f64 = + rest.trim().trim_end_matches("kB").trim().parse().unwrap_or(0.0); + kib / 1024.0 / 1024.0 +} + +/// One block of the static schedule: home address, member constants, +/// serialized size. +struct SchedBlock { + addr: Address, + members: Vec
, + size: u64, +} + +/// Fold the env's constants into home blocks (projections and mutual +/// members attribute to their block), with member lists, sizes, and the +/// block-level reference adjacency — all static structure, no execution. +fn schedule_blocks(env: &IxonEnv) -> (Vec, Vec>) { + // Pass 1: home address per constant. + let mut home: FxHashMap = FxHashMap::default(); + for entry in env.consts.iter() { + let (addr, lazy) = (entry.key(), entry.value()); + let Ok(c) = lazy.get() else { continue }; + let h = match &c.info { + IxonCI::IPrj(p) => p.block.clone(), + IxonCI::CPrj(p) => p.block.clone(), + IxonCI::RPrj(p) => p.block.clone(), + IxonCI::DPrj(p) => p.block.clone(), + _ => addr.clone(), + }; + home.insert(addr.clone(), h); + } + // Pass 2: block table (sorted for determinism) + members + sizes. + let mut blocks: FxHashMap = FxHashMap::default(); + for entry in env.consts.iter() { + let (addr, _lazy) = (entry.key(), entry.value()); + let h = home[addr].clone(); + let size = env.get_const_bytes(&h).map_or(0, |b| b.len() as u64); + let b = blocks.entry(h.clone()).or_insert_with(|| SchedBlock { + addr: h.clone(), + members: Vec::new(), + size, + }); + b.members.push(addr.clone()); + } + let mut list: Vec = blocks.into_values().collect(); + list.sort_by(|a, b| a.addr.cmp(&b.addr)); + for b in &mut list { + b.members.sort(); + } + let id_of: FxHashMap<&Address, u32> = list + .iter() + .enumerate() + .map(|(i, b)| (&b.addr, u32::try_from(i).expect("block count exceeds u32"))) + .collect(); + // Pass 3: block-level ref adjacency (constant refs folded to home blocks). + let mut adj: Vec> = vec![Vec::new(); list.len()]; + for entry in env.consts.iter() { + let (addr, lazy) = (entry.key(), entry.value()); + let Ok(c) = lazy.get() else { continue }; + let Some(&hid) = id_of.get(&home[addr]) else { continue }; + for r in &c.refs { + if let Some(rh) = home.get(r) + && let Some(&rid) = id_of.get(rh) + && rid != hid + { + adj[hid as usize].push(rid); + } + } + } + for row in &mut adj { + row.sort_unstable(); + row.dedup(); + } + (list, adj) +} + +/// The byte-weighted min-cut linearization of the block graph. Weights ride +/// the `intern` counter slot (the only op counter in the step-cost formula +/// that we can set to a pure byte value), nets are the ref edges; the +/// bisection keeps closure-overlapping blocks adjacent. +fn static_order( + blocks: &[SchedBlock], + adj: &[Vec], + pieces: usize, +) -> Vec { + let mut b = ProfileBuilder::new(); + for blk in blocks { + let ops = OpCounts { intern_nodes: blk.size, ..OpCounts::default() }; + b.block( + blk.addr.clone(), + 0, + u32::try_from(blk.size).expect("block size exceeds u32"), + u32::try_from(blk.members.len()).expect("member count exceeds u32"), + ops, + ); + } + for (i, row) in adj.iter().enumerate() { + for &r in row { + b.delta_edge(blocks[i].addr.clone(), blocks[r as usize].addr.clone()); + } + } + let profile = b.finish(); + // ProfileBuilder sorts by address and `blocks` is address-sorted, so ids + // coincide; assert the invariant the whole mapping rests on. + assert_eq!(profile.num_blocks(), blocks.len()); + cut_coherent_order(&profile, pieces, 0.05) +} + +/// One scanned segment: its blocks (as schedule ids) and BOTH measured +/// resource terms — FFT cost (prove compute/trace) and retained record +/// bytes (the execute-side store the prove replays into). Record bytes +/// are not derivable from FFT cost: measured GiB-per-BFFT varies ~2x +/// across segments (heavy single blocks run byte-lean, block-dense +/// segments byte-rich), so a RAM budget must price the pair. +struct Segment { + blocks: Vec, + fft: f64, + ram_gib: f64, + /// Per-circuit membership sketch of the segment's record — the union + /// pricing input. `None` in execute-only mode (no partition to price) + /// and on decode failures, where the merge falls back to summing this + /// segment conservatively. + sketch: Option, +} + +/// Everything a chunk scanner needs besides its own chunk: kernel, env, +/// schedule, and the cut threshold. +struct ScanCtx<'a> { + toplevel: &'a Toplevel, + fun_idx: usize, + env: &'a Arc, + blocks: &'a [SchedBlock], + /// Combined-resource cut: a segment ends when + /// `RAM_GIB_PER_BFFT·fft + record_bytes` reaches this headroom + /// (the budget's usable GiB above the prove base, ε-discounted). + cut_used_gib: f64, + n_chunks: usize, + /// Abort the whole scan on the first kernel-rejected block (the + /// default). With `--no-fail-fast`, such blocks are recorded in + /// `failed` and skipped; the manifest then does not cover them, which + /// the downstream coverage gate reports — a partial partition can + /// never pass as a full-env check. + fail_fast: bool, + /// Blocks whose check claim the kernel rejected, with the error. + failed: &'a Mutex>, + /// Set on a fatal error so every worker bails at its next BLOCK + /// boundary — a failure aborts the fleet in seconds, not after the + /// in-flight ranges drain. + abort: &'a std::sync::atomic::AtomicBool, + /// Blocks per measurement claim. Batching divides the claim layer's + /// per-claim overhead (assumption-tree hashing, unshared `env_walk` + /// frames) by K and shrinks frontiers (intra-batch edges stop being + /// frontier members), keeping the running readout tight enough to + /// serve as the shard cost without a blanket re-price. + batch_blocks: usize, + /// The compiled system, feeding the analytic peak-prove-RAM model that + /// the scan's cut charges (`Some` for the scanner); `None` in + /// execute-only mode, where the cut is the record's retained bytes + /// against the per-worker share. + system: Option<&'a AiurSystem>, + /// Execute-only phase 1: defer a range's remainder to the fat phase + /// when measured record growth per block exceeds this (bytes). + /// `None` disables (the scan, and the fat phase itself). + defer_growth: Option, + /// Graceful record ceiling, GiB: a segment also cuts when its record's + /// retained bytes reach this, independent of the model cut. In + /// cgroup-capped worker processes it sits below the enforced cap + /// (~80%), so legitimately record-heavy segments end at a claim + /// boundary instead of being OOM-killed — the kill is reserved for + /// MID-claim growth, which no boundary check can see. `f64::INFINITY` + /// disables it (thread-pool mode). + soft_record_gib: f64, +} + +/// Default blocks per measurement claim; `IX_SCAN_BATCH_BLOCKS` +/// overrides (1 restores per-block claims). Fixed — never adaptive — so +/// the partition stays independent of scheduling. The claim is also the +/// granularity of the record-threshold check, so its worst-case record +/// growth must fit inside [`CLAIM_HEADROOM_GIB`]: dense FLT content +/// measured ~4 GiB of growth per 128-block claim, scaling roughly with +/// K. K=32 bounds that to ~1 GiB at a conservative measurement drift +/// between the measured +3.7% (K=64) and +11.6% (K=16) — drift only +/// overstates costs, tightening the plan, never breaking it. +const SCAN_BATCH_BLOCKS: usize = 32; + +/// Target blocks per work-queue chunk. Chunks are the unit of stealing: +/// once fewer chunks remain than workers, the excess workers idle — the +/// serial tail is bounded by the largest chunk, so chunks must be much +/// finer than the worker count (workers×2 on FLT left 16 of 18 workers +/// idle for the last third of the scan). Chunk edges are forced segment +/// boundaries; the merge pass absorbs the extra fragmentation. +const CHUNK_TARGET_BLOCKS: usize = 2048; + +/// A worker's non-record residency: env mmap, compiled AiurSystem, +/// runtime (fresh workers measured ~1.2-1.8 GiB on FLT), PLUS working +/// room for the env decode cache, which grows monotonically with +/// content touched (death is its shedding mechanism — but it must not +/// be the routine one, so the slot budgets a few GiB of cache first). +const WORKER_BASELINE_GIB: f64 = 3.5; + +/// Target record growth per claim, GiB. Claim width K derives from it +/// (see [`scan_range`]): K = target / measured-growth-per-block, clamped +/// to `[1, SCAN_BATCH_BLOCKS]`. Per-claim overhead (assumption-tree +/// hashing, unshared `env_walk` frames) is roughly constant per claim, +/// so it is negligible against a claim carrying this much work — light +/// content runs at full width where the overhead would bite, dense +/// content shrinks to K=1-2 where each block dwarfs it. Bounding growth +/// per claim is what lets the between-claims threshold check act before +/// the cgroup kill on ANY content. +const CLAIM_TARGET_GIB: f64 = 0.75; + +/// Worst mid-claim record growth past the threshold check: with claim +/// width derived from [`CLAIM_TARGET_GIB`], overshoot beyond the target +/// is one block's excess over its range's running estimate — bounded in +/// practice by the single-block record distribution's body; the tail +/// (true monster blocks) is the deferred cleanup's job, not headroom's. +const CLAIM_HEADROOM_GIB: f64 = 1.0; + +/// Smallest useful per-worker slice: the soft record cut (the segment +/// measurement quantum, ~11.5 GiB at this floor — segments are summed +/// to the cut by the merge pass, so they never need to reach it alone) +/// plus the cache-inclusive worker baseline and one claim's headroom. +/// Bounds the auto worker count (`pool / floor`). Measured across the +/// FLT sweep: 8 GiB slots (60 workers) and 12 GiB slots (33 workers) +/// both ground down in the dense head — cache + a useful record quantum +/// simply need this much — while ~17-18 GiB slots ran it best; thinner +/// buys width the dense content immediately claws back in deaths. +const SLICE_FLOOR_GIB: f64 = 16.0; + +/// Width of the deferred-block cleanup round: blocks whose own claim +/// died under a slot cap re-run after the fleet drains, each worker +/// capped at the freed pool split this many ways (~65-70 GiB on the +/// deliverable boxes). Sized from the monster-record distribution: +/// every escalated block ever measured except two fit under ~70 GiB, +/// so a wider round would trade slots that fit the population for +/// parallelism the survivors cannot use. +const CLEANUP_WORKERS: usize = 12; + +/// Execute-only phase-1 deferral threshold: record growth per block +/// (bytes) above which a range's remainder is handed to the fat-slot +/// phase instead of walked thin. The measured distribution is bimodal +/// with a decade-wide gap (light content ~1-20 MB/block, dense +/// typeclass-web content ~300 MB-4 GiB/block), so any threshold in the +/// gap classifies robustly, and both error directions are cheap: an +/// over-deferred range walks warm in phase 2; an under-deferred one +/// triggers on its next claim. Growth is measured, not predicted — +/// record growth IS the reduction work just performed. +const DEFER_GROWTH_BYTES_PER_BLOCK: f64 = 1.0e8; + +/// Union pricing: summed segment costs overstate a merged shard's true +/// cold cost (shared cone queries count once per SEGMENT in the sum but +/// once per SHARD in the prove's single cold record — measured +/// true/summed = 0.56-0.75), and per-query cost is context-free, so the +/// exact fix is set semantics: a shard's true heights are the unique- +/// query counts of the UNION of its segments' records. Each segment +/// ships a per-circuit membership sketch of its record (the stored +/// 64-bit entry hashes — no rehashing): an EXACT hash list for maps up +/// to [`SKETCH_EXACT_MAX`] uniques, a HyperLogLog register array above. +/// The merge unions sketches instead of summing, feeds the estimated +/// union heights to the same analytic cost/RAM model, and cuts directly +/// at the budget — no overshoot factor, no cold re-price pass. +/// +/// HLL precision: `2^HLL_P` registers → σ ≈ 1.04/√2^P (~1.2% at P=13). +/// Estimates from HLL (never the exact lists) are inflated by +/// [`SKETCH_SAFETY`] (~2σ) so sketch noise errs conservative, and every +/// estimate is clamped to [max single-segment raw, Σ raws]. +const HLL_P: u32 = 13; +const HLL_REGS: usize = 1 << HLL_P; +const SKETCH_EXACT_MAX: usize = 1024; +const SKETCH_SAFETY: f64 = 1.025; + +/// `u64 → f64` without a lossy `as` cast (split into exactly-convertible +/// `u32` halves — the sketch magnitudes sit far below the 2^52 edge). +fn f64_from_u64(n: u64) -> f64 { + let hi = u32::try_from(n >> 32).expect("u64 is 64 bits"); + let lo = u32::try_from(n & 0xFFFF_FFFF).expect("masked to u32 range"); + f64::from(hi) * 4_294_967_296.0 + f64::from(lo) +} + +/// Saturating `f64 → u64` via decimal round-trip (no lossy `as`; runs a +/// handful of times per merge decision). +fn u64_from_f64(x: f64) -> u64 { + format!("{:.0}", x.max(0.0)).parse().unwrap_or(u64::MAX) +} + +/// splitmix64 finalizer: the stored map hashes are table-quality, not +/// avalanche-quality; HLL register selection and rank both need uniform +/// bits, so every hash is remixed on sketch insert (deterministic, so +/// partitions stay identical across runs and machines). +fn remix(mut x: u64) -> u64 { + x = (x ^ (x >> 30)).wrapping_mul(0xbf58476d1ce4e5b9); + x = (x ^ (x >> 27)).wrapping_mul(0x94d049bb133111eb); + x ^ (x >> 31) +} + +/// One query map's membership sketch: exact remixed hashes while small, +/// HLL registers above [`SKETCH_EXACT_MAX`]. +enum SketchSet { + Exact(Vec), + Hll(Box<[u8]>), +} + +impl SketchSet { + fn hll_insert(regs: &mut [u8], h: u64) { + let idx = usize::try_from(h >> (64 - HLL_P)).expect("HLL_P bits"); + let rank = u8::try_from((h << HLL_P | 1 << (HLL_P - 1)).leading_zeros()) + .expect("leading_zeros of u64 is at most 64") + + 1; + if regs[idx] < rank { + regs[idx] = rank; + } + } + + fn hll_estimate(regs: &[u8]) -> f64 { + let m = f64_from_usize(HLL_REGS); + let mut sum = 0.0f64; + let mut zeros = 0usize; + for &r in regs { + sum += 1.0 / f64_from_usize(1usize << r.min(63)); + if r == 0 { + zeros += 1; + } + } + let alpha = 0.7213 / (1.0 + 1.079 / m); + let e = alpha * m * m / sum; + if e <= 2.5 * m && zeros > 0 { + // Small-range correction: linear counting. + m * (m / f64_from_usize(zeros)).ln() + } else { + e + } + } +} + +/// One map's sketch entry: circuit key, measured raw uniques and +/// retained field elements (the record-bytes term scales with them), and +/// the membership set. +struct MapSketch { + /// 0 = function (id = function index), 1 = memory (id = width). + kind: u8, + id: u32, + raw: u64, + elems: u64, + set: SketchSet, +} + +/// A segment record's per-circuit membership sketches (every nonzero +/// query map, constrained or not — unconstrained maps carry no circuit +/// but do carry record bytes). +struct SegSketch { + maps: Vec, +} + +impl SegSketch { + fn of_record(record: &QueryRecord) -> Self { + let mut maps = Vec::new(); + let mut push = |kind: u8, id: u32, m: &aiur::querymap::QueryMap| { + let raw = m.len(); + if raw == 0 { + return; + } + let set = if raw <= SKETCH_EXACT_MAX { + let mut v = Vec::with_capacity(raw); + m.for_each_hash(|h| v.push(remix(h))); + v.sort_unstable(); + v.dedup(); + SketchSet::Exact(v) + } else { + let mut regs = vec![0u8; HLL_REGS].into_boxed_slice(); + m.for_each_hash(|h| SketchSet::hll_insert(&mut regs, remix(h))); + SketchSet::Hll(regs) + }; + maps.push(MapSketch { + kind, + id, + raw: u64::try_from(raw).unwrap_or(u64::MAX), + elems: u64::try_from(m.retained_elems()).unwrap_or(u64::MAX), + set, + }); + }; + for (i, m) in record.function_queries.iter().enumerate() { + push(0, u32::try_from(i).unwrap_or(u32::MAX), m); + } + for (&w, m) in &record.memory_queries { + push(1, u32::try_from(w).unwrap_or(u32::MAX), m); + } + SegSketch { maps } + } + + fn to_bytes(&self) -> Vec { + let mut out = Vec::new(); + out.extend_from_slice( + &u32::try_from(self.maps.len()).unwrap_or(0).to_le_bytes(), + ); + for m in &self.maps { + out.push(m.kind); + out.extend_from_slice(&m.id.to_le_bytes()); + out.extend_from_slice(&m.raw.to_le_bytes()); + out.extend_from_slice(&m.elems.to_le_bytes()); + match &m.set { + SketchSet::Exact(v) => { + out.push(0); + out.extend_from_slice( + &u32::try_from(v.len()).unwrap_or(0).to_le_bytes(), + ); + for h in v { + out.extend_from_slice(&h.to_le_bytes()); + } + }, + SketchSet::Hll(regs) => { + out.push(1); + out.extend_from_slice(regs); + }, + } + } + out + } + + fn from_bytes(b: &[u8]) -> Option { + let mut p = 0usize; + let take = |p: &mut usize, n: usize| -> Option<&[u8]> { + let s = b.get(*p..*p + n)?; + *p += n; + Some(s) + }; + let n = u32::from_le_bytes(take(&mut p, 4)?.try_into().ok()?) as usize; + let mut maps = Vec::with_capacity(n); + for _ in 0..n { + let kind = take(&mut p, 1)?[0]; + let id = u32::from_le_bytes(take(&mut p, 4)?.try_into().ok()?); + let raw = u64::from_le_bytes(take(&mut p, 8)?.try_into().ok()?); + let elems = u64::from_le_bytes(take(&mut p, 8)?.try_into().ok()?); + let set = match take(&mut p, 1)?[0] { + 0 => { + let k = + u32::from_le_bytes(take(&mut p, 4)?.try_into().ok()?) as usize; + let mut v = Vec::with_capacity(k); + for _ in 0..k { + v.push(u64::from_le_bytes(take(&mut p, 8)?.try_into().ok()?)); + } + SketchSet::Exact(v) + }, + 1 => SketchSet::Hll(take(&mut p, HLL_REGS)?.to_vec().into_boxed_slice()), + _ => return None, + }; + maps.push(MapSketch { kind, id, raw, elems, set }); + } + Some(SegSketch { maps }) + } +} + +/// Running union of segment sketches over one candidate shard. +#[derive(Clone)] +enum AccSet { + Exact(std::collections::HashSet), + Hll(Box<[u8]>), +} + +#[derive(Clone)] +struct MapAcc { + raw_sum: u64, + max_raw: u64, + /// Retained field elements per unique entry (fixed per map: key and + /// output strides are per-circuit constants). + elems_per_raw: f64, + set: AccSet, +} + +impl MapAcc { + fn absorb(&mut self, s: &MapSketch) { + self.raw_sum += s.raw; + self.max_raw = self.max_raw.max(s.raw); + match (&mut self.set, &s.set) { + (AccSet::Exact(acc), SketchSet::Exact(v)) => acc.extend(v.iter()), + (AccSet::Hll(regs), SketchSet::Hll(r)) => { + for (a, b) in regs.iter_mut().zip(r.iter()) { + *a = (*a).max(*b); + } + }, + (AccSet::Exact(acc), SketchSet::Hll(r)) => { + let mut regs = r.clone(); + for &h in acc.iter() { + SketchSet::hll_insert(&mut regs, h); + } + self.set = AccSet::Hll(regs); + }, + (AccSet::Hll(regs), SketchSet::Exact(v)) => { + for &h in v { + SketchSet::hll_insert(regs, h); + } + }, + } + } + + /// Estimated unique entries of the union: exact while every + /// contribution stayed exact; HLL (safety-inflated) once any map went + /// probabilistic; always within [max_raw, raw_sum]. + fn estimate(&self) -> u64 { + let est = match &self.set { + AccSet::Exact(s) => u64::try_from(s.len()).unwrap_or(u64::MAX), + AccSet::Hll(regs) => { + u64_from_f64(SketchSet::hll_estimate(regs) * SKETCH_SAFETY) + }, + }; + est.clamp(self.max_raw, self.raw_sum) + } +} + +#[derive(Clone, Default)] +struct UnionAcc { + maps: std::collections::HashMap<(u8, u32), MapAcc>, + /// Segments that arrived without a sketch (worker of an older vintage, + /// or a decode failure): their summed (fft, ram) is added on top of + /// the union model — conservative, never unsound. + unsketched_fft: f64, + unsketched_ram: f64, + n_segments: usize, +} + +impl UnionAcc { + fn absorb(&mut self, seg: &Segment) { + self.n_segments += 1; + match &seg.sketch { + Some(sk) => { + for m in &sk.maps { + let acc = + self.maps.entry((m.kind, m.id)).or_insert_with(|| MapAcc { + raw_sum: 0, + max_raw: 0, + elems_per_raw: if m.raw > 0 { + f64_from_u64(m.elems) / f64_from_u64(m.raw) + } else { + 0.0 + }, + set: match &m.set { + SketchSet::Exact(_) => { + AccSet::Exact(std::collections::HashSet::new()) + }, + SketchSet::Hll(_) => { + AccSet::Hll(vec![0u8; HLL_REGS].into_boxed_slice()) + }, + }, + }); + acc.absorb(m); + } + }, + None => { + self.unsketched_fft += seg.fft; + self.unsketched_ram += seg.ram_gib; + }, + } + } + + /// Model the union: estimated per-map unique heights + retained + /// elements → the same analytic (fft, peak-prove-RAM) pair a single + /// cold record of the union would produce, plus the summed + /// contribution of unsketched segments. + fn model(&self, system: &AiurSystem) -> (f64, f64) { + let ests: Vec<(bool, usize, usize, usize)> = self + .maps + .iter() + .map(|((kind, id), acc)| { + let est = acc.estimate(); + let elems = usize::try_from(u64_from_f64( + f64_from_u64(est) * acc.elems_per_raw, + )) + .unwrap_or(usize::MAX); + (*kind == 0, *id as usize, usize::try_from(est).unwrap_or(usize::MAX), elems) + }) + .collect(); + let (fft, peak) = system.model_from_map_estimates(&ests); + ( + fft + self.unsketched_fft, + f64_from_usize(peak.peak) / GIB + self.unsketched_ram, + ) + } +} + +/// Ranges a child serves before the parent proactively reaps and +/// respawns it. The env decode cache grows monotonically with content +/// touched — measured ~2-4 GiB per DENSE range — so recycling bounds +/// every child's cache to about one range's growth: slot headroom +/// belongs to the RECORD at every point in the schedule, dense strips +/// walk inline at fleet width, and mid-strip deaths (whose resume +/// points cannot re-open under a slot and would push whole strip +/// remainders into the narrow cleanup round) stay rare. Spawn cost is +/// seconds (order file + env mmap) against tens of seconds per range. +const WORKER_RECYCLE_RANGES: usize = 2; + +/// Subtracted from box RAM (with the measured parent baseline) before +/// slicing the pool: kernel, page cache churn, and everything else on +/// the box that is not this scan. +const OS_RESERVE_GIB: f64 = 12.0; + +/// Fraction of the derived pool actually sliced into worker caps. The +/// caps are a worst-case bound and dense regions reach it: on FLT's +/// head every worker sits near its cap simultaneously, so `Σ caps = +/// pool` runs the box to zero free (measured: 490/495 GB used, page +/// cache evicted to zero). The unsliced remainder is the fleet's slack — +/// all workers brushing their caps at once still leave it free. +const POOL_SLICE_FRAC: f64 = 0.85; + +/// Worker counts shrink until every child's even slice of the pool +/// clears the slice floor; thread mode passes through. +fn bound_workers_by_pool(workers: usize, pool_gib: f64, proc: bool) -> usize { + if !proc { + return workers; + } + let by_floor: usize = format!("{:.0}", (pool_gib / SLICE_FLOOR_GIB).floor()) + .parse() + .unwrap_or(1); + workers.min(by_floor.max(1)).max(1) +} + +/// The min-cut schedule order, windowed by the `IX_SCAN_SKIP_BLOCKS` / +/// `IX_SCAN_LIMIT_BLOCKS` debug knobs (a full-pipeline reproducer over a +/// slice of a huge env, without extracting one; the result then does NOT +/// cover the env). Skip drops the order's head, limit truncates what +/// remains — composed, they select any window; skip alone replays the +/// schedule's TAIL, where min-cut ordering concentrates the dense +/// content that dominates scan wall. +fn ordered_schedule( + blocks: &[SchedBlock], + adj: &[Vec], + n_chunks: usize, +) -> Vec { + let mut order = static_order(blocks, adj, n_chunks.max(16)); + if let Some(skip) = std::env::var("IX_SCAN_SKIP_BLOCKS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&n| n > 0) + { + let skip = skip.min(order.len()); + eprintln!( + "[scan] IX_SCAN_SKIP_BLOCKS={skip}: executing a schedule SUFFIX — \ + the result will not cover the env" + ); + order.drain(..skip); + } + if let Some(limit) = std::env::var("IX_SCAN_LIMIT_BLOCKS") + .ok() + .and_then(|v| v.parse::().ok()) + && limit < order.len() + { + eprintln!( + "[scan] IX_SCAN_LIMIT_BLOCKS={limit}: executing a schedule PREFIX — \ + the result will not cover the env" + ); + order.truncate(limit); + } + order +} + +/// Blocks per chunk past which an edge is forced regardless of bytes. +/// Chunks are byte-balanced, but cost per byte is wildly non-uniform: +/// FLT's dense proof blocks are small in bytes and huge in cost, so +/// byte-only balancing packed up to ~3,100 of them into one chunk — +/// ~10 sequential cut-sized segments for a single owner, the measured +/// serial tail of the scan. The count clamp splits exactly those +/// chunks; the merge pass absorbs the extra forced boundaries. +const CHUNK_MAX_BLOCKS: usize = 512; + +/// Equal-byte contiguous chunks over the order, block-count clamped +/// (see [`CHUNK_MAX_BLOCKS`]); edges are forced segment boundaries (the +/// parallelism unit); the scan's merge pass repairs the resulting +/// fragmentation, so chunk granularity is a pure parallelism knob. +fn make_chunk_bounds( + order: &[u32], + blocks: &[SchedBlock], + env_bytes: u64, + n_chunks: usize, +) -> Vec<(usize, usize)> { + let per_chunk = (env_bytes / n_chunks as u64).max(1); + let mut bounds: Vec<(usize, usize)> = Vec::new(); + let mut start = 0usize; + let mut acc = 0u64; + for (i, &b) in order.iter().enumerate() { + acc += blocks[b as usize].size; + if acc >= per_chunk || i + 1 - start >= CHUNK_MAX_BLOCKS { + bounds.push((start, i + 1)); + start = i + 1; + acc = 0; + } + } + if start < order.len() { + bounds.push((start, order.len())); + } + // Baseline before any execution: the schedule pass decoded every + // constant into the shared env's lazy cache, so this RSS is (cache + + // static structures) — the floor the worker footprints sit on. + eprintln!("[scan] post-schedule baseline rss {:.0}G", process_rss_gib()); + bounds +} + +/// Work-stealing worker pool over the chunks: workers pull `(origin +/// chunk, seq, blocks)` ranges off a shared deque; a range yields at +/// most [`RANGE_SEGMENTS`] segments, then re-queues its remainder for +/// any idle worker, so dense regions self-parallelize. The split policy +/// is count-based, not time- or RAM-based, so the resulting segments do +/// not depend on scheduling; they are tagged `(origin, seq)` and sorted +/// at the end, so the returned order is the schedule order. +fn run_pool( + ctx: &ScanCtx<'_>, + chunks: Vec>, + workers: usize, +) -> Result, String> { + type Range = (u32, u32, Vec); + let queue: Mutex> = Mutex::new( + chunks + .into_iter() + .enumerate() + .map(|(i, c)| (u32::try_from(i).expect("chunk count fits u32"), 0u32, c)) + .collect(), + ); + let in_flight = AtomicUsize::new(0); + let done: Mutex)>> = Mutex::new(Vec::new()); + let failure: Mutex> = Mutex::new(None); + std::thread::scope(|s| { + for _ in 0..workers { + s.spawn(|| { + loop { + if failure.lock().unwrap().is_some() { + break; + } + // Pop and the in-flight increment are ATOMIC under the queue + // lock: a worker that sees the queue empty is then guaranteed a + // consistent in-flight read — the last popper has already + // registered. (Split, the window between pop and increment let + // idle workers read `empty && in_flight == 0` and exit while + // work remained; the fleet silently drained.) + let next = { + let mut q = queue.lock().unwrap(); + let popped = q.pop_front(); + if popped.is_some() { + in_flight.fetch_add(1, Ordering::AcqRel); + } + popped + }; + let Some((origin, seq, range)) = next else { + // Empty queue but ranges in flight may still re-queue + // remainders; only quit when nothing can produce more work. + if in_flight.load(Ordering::Acquire) == 0 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + continue; + }; + match scan_range(ctx, &range, origin) { + Ok((segs, rest, _defer)) => { + done.lock().unwrap().push(((origin, seq), segs)); + // Remainder goes back BEFORE the in-flight decrement, so + // `empty && in_flight == 0` still implies no future work. + if !rest.is_empty() { + queue.lock().unwrap().push_back((origin, seq + 1, rest)); + } + }, + Err(e) => { + ctx.abort.store(true, Ordering::Release); + let mut f = failure.lock().unwrap(); + if f.is_none() { + *f = Some(e); + } + }, + } + in_flight.fetch_sub(1, Ordering::AcqRel); + } + }); + } + }); + if let Some(e) = failure.into_inner().unwrap() { + return Err(e); + } + let mut tagged = done.into_inner().unwrap(); + tagged.sort_by_key(|(k, _)| *k); + let mut segments: Vec = Vec::new(); + for (_, mut segs) in tagged { + segments.append(&mut segs); + } + Ok(segments) +} + +/// The scan worker's stdin/stdout loop: a child process spawned by the +/// process pool, deterministically re-deriving the same schedule as its +/// parent and executing order-index ranges on command. Line protocol +/// (one command per stdin line, replies on stdout): +/// +/// - `SCAN [narrow]` — scan `order[lo..hi)` exactly like a +/// thread worker's range: up to [`RANGE_SEGMENTS`] segments, then hand +/// the remainder back. The optional `narrow` count single-steps the +/// range's first N blocks (one per claim) — sent by the parent when a +/// range resumes after a death so a dense stretch banks per-block +/// progress instead of re-dying at full claim width. Replies: +/// `SEG ` per emitted segment (absolute order +/// indices), `SKIP ` per kernel-rejected block, +/// then `END ` (`next == hi` when the range is exhausted). A +/// unit range (`hi == lo+1`) degenerates to a single-block claim — +/// the deferred-block cleanup round measures blocks this way, no +/// dedicated verb needed. +/// +/// The worker never applies fail-fast itself — it reports SKIPs and the +/// parent enforces policy. RAM: the enclosing cgroup's `memory.max` is +/// the hard cap (an over-cap worker is OOM-killed and the parent +/// recovers); `soft_record_gib` cuts segments gracefully below it so +/// only MID-claim growth ever reaches the kill. +pub fn scan_worker( + system: &AiurSystem, + fun_idx: usize, + env: &Arc, + cut_used_gib: f64, + batch_blocks: usize, + soft_record_gib: f64, + pieces: usize, + exec_only: bool, + defer_growth: bool, +) -> Result<(), String> { + use std::io::{BufRead, Write}; + let toplevel = system.toplevel(); + // The parent already derived the schedule; children read it from the + // order file instead of re-deriving (30 children re-running the + // min-cut bisection concurrently was minutes of pure startup on + // FLT-scale envs). Fallback to self-derivation keeps the worker + // usable standalone. + let (blocks, order) = match std::env::var("IX_SCAN_ORDER_FILE") { + Ok(path) => read_order_file(&path)?, + Err(_) => { + let (blocks, adj) = schedule_blocks(env); + let order = ordered_schedule(&blocks, &adj, pieces); + (blocks, order) + }, + }; + let failed: Mutex> = Mutex::new(Vec::new()); + let abort = std::sync::atomic::AtomicBool::new(false); + let ctx = ScanCtx { + toplevel, + fun_idx, + env, + blocks: &blocks, + cut_used_gib, + n_chunks: pieces, + fail_fast: false, + failed: &failed, + abort: &abort, + batch_blocks, + system: if exec_only { None } else { Some(system) }, + defer_growth: defer_growth.then_some(DEFER_GROWTH_BYTES_PER_BLOCK), + soft_record_gib, + }; + let stdout = std::io::stdout(); + let mut out = stdout.lock(); + writeln!(out, "READY {}", order.len()).map_err(|e| e.to_string())?; + out.flush().map_err(|e| e.to_string())?; + let stdin = std::io::stdin(); + for line in stdin.lock().lines() { + let line = line.map_err(|e| e.to_string())?; + let mut it = line.split_whitespace(); + let (verb, lo, hi) = ( + it.next().unwrap_or(""), + it.next().and_then(|v| v.parse::().ok()), + it.next().and_then(|v| v.parse::().ok()), + ); + // A third field (a since-removed narrow-prefix hint) is accepted + // and ignored for wire compatibility with parents of that vintage. + let _ = it.next(); + let (Some(lo), Some(hi)) = (lo, hi) else { + return Err(format!("worker: malformed command {line:?}")); + }; + if hi > order.len() || lo >= hi { + return Err(format!("worker: range {lo}..{hi} out of bounds")); + } + match verb { + "SCAN" => { + let range = &order[lo..hi]; + let (segs, rest, deferred) = + scan_range(&ctx, range, u32::try_from(lo).unwrap_or(0))?; + // Segments consume the range in order; skipped blocks sit in the + // gaps. Recover absolute bounds by cursor-matching first ids. + let mut cursor = lo; + for s in &segs { + while order[cursor] != s.blocks[0] { + cursor += 1; // a skipped block + } + let end = cursor + s.blocks.len(); + // Fifth field: the segment's membership sketch (hex), the + // union-pricing input. Sized by the record's nonzero maps + // (small maps ship exact hash lists, big ones fixed HLL + // registers) — hundreds of KB against a multi-second segment. + match &s.sketch { + Some(sk) => writeln!( + out, + "SEG {cursor} {end} {} {} {}", + s.fft, + s.ram_gib, + hex_encode(&sk.to_bytes()) + ), + None => { + writeln!(out, "SEG {cursor} {end} {} {}", s.fft, s.ram_gib) + }, + } + .map_err(|e| e.to_string())?; + cursor = end; + } + for (a, e) in failed.lock().unwrap().drain(..) { + writeln!(out, "SKIP {} {}", a.hex(), hex_encode(e.as_bytes())) + .map_err(|e| e.to_string())?; + } + let verb = if deferred { "DEFER" } else { "END" }; + writeln!(out, "{verb} {}", hi - rest.len()) + .map_err(|e| e.to_string())?; + }, + _ => return Err(format!("worker: unknown verb {verb:?}")), + } + out.flush().map_err(|e| e.to_string())?; + } + Ok(()) +} + +/// Serialize the derived schedule for worker children: block addresses +/// (in block-id order) and the min-cut order. Members and sizes are +/// parent-side concerns (chunking, manifest assembly) — a worker needs +/// only `id → addr` for claims and the order for range indexing. +fn write_order_file( + path: &std::path::Path, + blocks: &[SchedBlock], + order: &[u32], +) -> Result<(), String> { + let mut buf: Vec = + Vec::with_capacity(16 + blocks.len() * 32 + order.len() * 4); + buf.extend_from_slice(&(blocks.len() as u64).to_le_bytes()); + for b in blocks { + buf.extend_from_slice(b.addr.as_bytes()); + } + buf.extend_from_slice(&(order.len() as u64).to_le_bytes()); + for &o in order { + buf.extend_from_slice(&o.to_le_bytes()); + } + std::fs::write(path, buf).map_err(|e| format!("write {path:?}: {e}")) +} + +fn read_order_file(path: &str) -> Result<(Vec, Vec), String> { + let buf = + std::fs::read(path).map_err(|e| format!("read order file {path}: {e}"))?; + let take_u64 = |buf: &[u8], pos: usize| -> Result { + buf + .get(pos..pos + 8) + .and_then(|b| b.try_into().ok()) + .map(u64::from_le_bytes) + .ok_or_else(|| format!("order file {path}: truncated")) + }; + let nblocks = usize::try_from(take_u64(&buf, 0)?) + .map_err(|_e| "order file: block count overflow".to_string())?; + let mut pos = 8; + let mut blocks = Vec::with_capacity(nblocks); + for _ in 0..nblocks { + let addr = buf + .get(pos..pos + 32) + .and_then(|b| Address::from_slice(b).ok()) + .ok_or_else(|| format!("order file {path}: truncated address"))?; + blocks.push(SchedBlock { addr, members: Vec::new(), size: 0 }); + pos += 32; + } + let norder = usize::try_from(take_u64(&buf, pos)?) + .map_err(|_e| "order file: order count overflow".to_string())?; + pos += 8; + let mut order = Vec::with_capacity(norder); + for _ in 0..norder { + let v = buf + .get(pos..pos + 4) + .and_then(|b| b.try_into().ok()) + .map(u32::from_le_bytes) + .ok_or_else(|| format!("order file {path}: truncated order"))?; + order.push(v); + pos += 4; + } + Ok((blocks, order)) +} + +fn hex_encode(bytes: &[u8]) -> String { + bytes.iter().map(|b| format!("{b:02x}")).collect() +} + +fn hex_decode(s: &str) -> Option> { + if !s.len().is_multiple_of(2) { + return None; + } + (0..s.len() / 2) + .map(|i| u8::from_str_radix(&s[2 * i..2 * i + 2], 16).ok()) + .collect() +} + +/// Whether `systemd-run --user --scope` is available to enforce +/// per-worker memory caps (the kernel migration a session-scoped process +/// cannot do itself, done by the user manager over D-Bus). Probed once +/// per pool; without it workers run uncapped with a loud warning. +fn systemd_scope_caps_available() -> bool { + std::process::Command::new("systemd-run") + .args(["--user", "--scope", "--quiet", "--", "true"]) + .status() + .map(|st| st.success()) + .unwrap_or(false) +} + +/// Everything the process pool needs to spawn and command workers. +struct ProcPool<'a> { + bin: String, + ixe: String, + cut_used_gib: f64, + batch_blocks: usize, + soft_record_gib: f64, + pieces: usize, + exec_only: bool, + cap_bytes: u64, + /// Per-worker cap of the deferred-block cleanup round (the drained + /// pool split [`CLEANUP_WORKERS`] ways): after the fleet finishes, + /// blocks that could not run under a slot cap re-run through the same + /// pool code under these fat caps; survivors are named + /// resource-infeasible. + cleanup_cap_bytes: u64, + /// True in the cleanup round itself — a block failing there is named + /// infeasible instead of deferred again. + cleanup: bool, + /// The parent-derived schedule serialized for children (deleted when + /// the pool drops) — spawn startup is env-mmap + system build, not a + /// re-derivation. + order_file: std::path::PathBuf, + /// Caps are enforced via `systemd-run --user --scope -p MemoryMax=`; + /// false means the probe failed and workers run UNCAPPED. + capped: bool, + order: &'a [u32], + blocks: &'a [SchedBlock], + fail_fast: bool, + /// Name deferred ranges resource-infeasible instead of walking them + /// in the cleanup round (`ix shard --defer-infeasible`). + defer_infeasible: bool, + /// Execute-only phase 1: children measure growth and hand dense + /// remainders back (`DEFER`) for the fat phase. + defer_growth: bool, +} + +struct WorkerHandle { + child: std::process::Child, + stdin: std::process::ChildStdin, + stdout: std::io::BufReader, +} + +/// One `SCAN` round-trip's outcome. +struct ScanReply { + segs: Vec, + skips: Vec<(Address, String)>, + next: usize, + /// The worker stopped at `next` because measured growth crossed the + /// phase-1 threshold — the remainder belongs to the fat phase. + deferred: bool, +} + +impl Drop for ProcPool<'_> { + fn drop(&mut self) { + let _ = std::fs::remove_file(&self.order_file); + } +} + +impl ProcPool<'_> { + /// Spawn with one retry: a fresh worker that dies before its READY + /// handshake is a transient environment hiccup (systemd/D-Bus under + /// respawn churn measured one EOF in ~100 respawns), not a scan + /// failure — but an unhandled one aborts the whole scan. One backoff + /// retry covers it; a second failure is real and propagates. + fn spawn(&self, slot: usize) -> Result { + self.spawn_once(slot).or_else(|e| { + eprintln!("[scan] worker {slot} spawn failed ({e}); retrying once"); + std::thread::sleep(std::time::Duration::from_secs(2)); + self.spawn_once(slot) + }) + } + + fn spawn_once(&self, slot: usize) -> Result { + use std::process::{Command, Stdio}; + let cap_bytes = self.cap_bytes; + let mut cmd = if self.capped { + let mut c = Command::new("systemd-run"); + c.args([ + "--user", + "--scope", + "--quiet", + "-p", + &format!("MemoryMax={cap_bytes}"), + "-p", + "MemorySwapMax=0", + "--", + &self.bin, + ]); + c + } else { + Command::new(&self.bin) + }; + cmd + .arg("shard-worker") + .env("IX_SCAN_ORDER_FILE", &self.order_file); + cmd + // Return freed pages to the OS immediately: the record drops at + // every segment cut, but mimalloc retains the pages by default, so + // worker RSS ratchets to its per-segment peak and the fleet sits + // at Σ caps regardless of live bytes. + .env("MIMALLOC_PURGE_DELAY", "0") + .args(["--ixe", &self.ixe]) + .args(["--cut-gib", &format!("{}", self.cut_used_gib)]) + .args(["--batch", &format!("{}", self.batch_blocks)]) + .args(["--soft-cap-gib", &format!("{}", self.soft_record_gib)]) + .args(["--pieces", &format!("{}", self.pieces)]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()); + if self.exec_only { + cmd.arg("--exec-only"); + } + if self.defer_growth { + cmd.arg("--defer-growth"); + } + let mut child = cmd + .spawn() + .map_err(|e| format!("spawn worker {slot} ({}): {e}", self.bin))?; + let stdin = child.stdin.take().expect("piped stdin"); + let stdout = + std::io::BufReader::new(child.stdout.take().expect("piped stdout")); + let mut h = WorkerHandle { child, stdin, stdout }; + // Verify the child derived the same schedule before trusting indices. + let ready = h.read_line()?; + let n: usize = ready + .strip_prefix("READY ") + .and_then(|v| v.trim().parse().ok()) + .ok_or_else(|| format!("worker {slot}: bad handshake {ready:?}"))?; + if n != self.order.len() { + return Err(format!( + "worker {slot}: schedule mismatch ({n} blocks vs {})", + self.order.len() + )); + } + Ok(h) + } + + /// Send one `SCAN` and collect its replies. `Err(committed)` = the + /// worker died mid-range; `committed` carries whatever segments and + /// skips arrived before death plus the index scanning had reached. + fn scan( + &self, + h: &mut WorkerHandle, + lo: usize, + hi: usize, + ) -> Result { + use std::io::Write; + let mut reply = ScanReply { + segs: Vec::new(), + skips: Vec::new(), + next: lo, + deferred: false, + }; + if writeln!(h.stdin, "SCAN {lo} {hi}").is_err() || h.stdin.flush().is_err() + { + return Err(reply); + } + loop { + let line = match h.read_line() { + Ok(l) => l, + Err(_) => return Err(reply), + }; + let mut it = line.split_whitespace(); + match it.next() { + Some("SEG") => { + let (Some(s), Some(e), Some(fft), Some(ram)) = ( + it.next().and_then(|v| v.parse::().ok()), + it.next().and_then(|v| v.parse::().ok()), + it.next().and_then(|v| v.parse::().ok()), + it.next().and_then(|v| v.parse::().ok()), + ) else { + return Err(reply); + }; + // Optional fifth field: the segment's membership sketch. A + // missing or undecodable sketch degrades that segment to + // conservative summing at the merge, never to an error. + let sketch = it + .next() + .and_then(hex_decode) + .and_then(|b| SegSketch::from_bytes(&b)); + reply.segs.push(Segment { + blocks: self.order[s..e].to_vec(), + fft, + ram_gib: ram, + sketch, + }); + reply.next = e; + }, + Some("SKIP") => { + let (Some(addr), Some(msg)) = ( + it.next().and_then(Address::from_hex), + it.next().and_then(hex_decode), + ) else { + return Err(reply); + }; + reply.skips.push((addr, String::from_utf8_lossy(&msg).into_owned())); + }, + Some("END") => { + let Some(next) = it.next().and_then(|v| v.parse::().ok()) + else { + return Err(reply); + }; + reply.next = next; + return Ok(reply); + }, + Some("DEFER") => { + let Some(next) = it.next().and_then(|v| v.parse::().ok()) + else { + return Err(reply); + }; + reply.next = next; + reply.deferred = true; + return Ok(reply); + }, + _ => return Err(reply), + } + } + } +} + +impl WorkerHandle { + fn read_line(&mut self) -> Result { + use std::io::BufRead; + let mut line = String::new(); + match self.stdout.read_line(&mut line) { + Ok(0) => Err("worker EOF".to_string()), + Ok(_) => Ok(line.trim_end().to_string()), + Err(e) => Err(e.to_string()), + } + } + + fn reap(mut self) -> String { + let _ = self.child.kill(); + match self.child.wait() { + Ok(st) => format!("{st}"), + Err(e) => format!("wait failed: {e}"), + } + } +} + +/// Process-pool scan: like [`run_pool`], but each worker is a separate +/// `ix shard-worker` process under a cgroup memory cap. A worker's env +/// decode cache grows monotonically with the content it executes (the +/// record drops at segment cuts; the cache never shrinks), so on dense +/// content every worker periodically fills its cap and is OOM-killed — +/// death IS the cache-shedding mechanism, and it is cheap: segments +/// stream as they close, so a kill loses only the work since the last +/// closed segment. The parent respawns and resumes from the committed +/// index with a narrow prefix (the resumed range's first blocks execute +/// one per claim), so a dense stretch banks per-block progress instead +/// of re-dying at full claim width. A block whose own single claim dies +/// under the slot cap is DEFERRED: after the fleet drains, the deferred +/// blocks re-run through this same function under fat caps (the freed +/// pool split [`CLEANUP_WORKERS`] ways); a block that dies even there is +/// named resource-infeasible. The fleet's RAM bound is `Σ caps`, +/// enforced by the kernel, independent of content. +fn run_pool_procs( + pool: &ProcPool<'_>, + chunks: Vec<(usize, usize)>, + workers: usize, + failed: &Mutex>, +) -> Result, String> { + // (origin chunk, commit sequence, lo, hi). + type Range = (u32, u32, usize, usize); + let total_blocks: usize = chunks.iter().map(|(lo, hi)| hi - lo).sum(); + let start = std::time::Instant::now(); + let queue: Mutex> = Mutex::new( + chunks + .into_iter() + .enumerate() + .map(|(i, (lo, hi))| { + (u32::try_from(i).expect("chunk count fits u32"), 0u32, lo, hi) + }) + .collect(), + ); + let in_flight = AtomicUsize::new(0); + let blocks_done = AtomicUsize::new(0); + let last_pct = AtomicUsize::new(0); + let done: Mutex)>> = Mutex::new(Vec::new()); + let failure: Mutex> = Mutex::new(None); + let abort = std::sync::atomic::AtomicBool::new(false); + // (origin, lo, hi) ranges whose opening claim died on a fresh worker — + // walked cumulatively in the cleanup round after the fleet drains. + // Deep dense strips are only cheap CUMULATIVELY (a walk shares the + // strip's dependency cone in one record; any solo measurement pays the + // whole cone per block), so a range whose resume point cannot even + // open under a slot moves to the fat round wholesale instead of the + // fleet paying one doomed cone-derivation per block. + let deferred: Mutex> = Mutex::new(Vec::new()); + std::thread::scope(|s| { + let (queue, in_flight, done, failure, abort) = + (&queue, &in_flight, &done, &failure, &abort); + let (blocks_done, last_pct, deferred) = (&blocks_done, &last_pct, &deferred); + for slot in 0..workers { + s.spawn(move || { + // Ranges served by the current child; at [`WORKER_RECYCLE_RANGES`] + // the parent reaps and respawns it proactively. + let mut served = 0usize; + // True until the current child completes its first scan: only a + // FRESH child's zero-progress death convicts a block — an aged + // child dying on a range's first claim indicts its own decode + // cache, not the block (348 false deferrals measured before this + // distinction; the requeued range simply waits for a fresh + // owner). + let mut fresh = true; + let mut worker = match pool.spawn(slot) { + Ok(w) => w, + Err(e) => { + let mut f = failure.lock().unwrap(); + if f.is_none() { + *f = Some(e); + } + abort.store(true, Ordering::Release); + return; + }, + }; + // Monotonic committed-block count; one line per percent crossed. + let progress = |n: usize| { + if n == 0 { + return; + } + let d = blocks_done.fetch_add(n, Ordering::AcqRel) + n; + let pct = d * 100 / total_blocks.max(1); + if pct > last_pct.fetch_max(pct, Ordering::AcqRel) { + eprintln!( + "[scan] {d}/{total_blocks} blocks ({pct}%), {:.0}s", + start.elapsed().as_secs_f64() + ); + } + }; + let commit = + |reply: ScanReply, origin: u32, seq: u32| -> Result { + if !reply.segs.is_empty() { + done.lock().unwrap().push(((origin, seq), reply.segs)); + } + if !reply.skips.is_empty() { + let fatal = pool.fail_fast; + let first = reply.skips.first().cloned(); + failed.lock().unwrap().extend(reply.skips); + if fatal { + if let Some((a, e)) = first { + let mut f = failure.lock().unwrap(); + if f.is_none() { + *f = Some(format!( + "CheckEnv of block {} failed during scan: {e} \ + (--no-fail-fast records and skips such blocks)", + a.hex() + )); + } + } + return Err(()); + } + } + Ok(reply.next) + }; + // Remainders go to the queue FRONT: a healthy remainder is + // usually re-popped by the worker that just banked its segments + // (cache still warm with the region's cone), and a death's + // remainder retries while its neighborhood is warm — pushed to + // the back, dense-region remainders sank behind hundreds of + // chunks and resurfaced 20 minutes later on cold workers, which + // died again and deferred the region wholesale. + let requeue = |origin: u32, seq: u32, lo: usize, hi: usize| { + if lo < hi { + queue.lock().unwrap().push_front((origin, seq, lo, hi)); + } + }; + loop { + if abort.load(Ordering::Acquire) { + break; + } + let next = { + let mut q = queue.lock().unwrap(); + let popped = q.pop_front(); + if popped.is_some() { + in_flight.fetch_add(1, Ordering::AcqRel); + } + popped + }; + let Some((origin, seq, lo, hi)) = next else { + if in_flight.load(Ordering::Acquire) == 0 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + continue; + }; + // Proactive recycle between ranges: the env decode cache grows + // monotonically and never shrinks, so a long-lived child ages + // toward its cap until ANY dense block kills it — a full-scan + // fleet reaching the dense tail with saturated caches misread + // the whole zone as monsters (161 false deferrals; the same + // tail scanned by young workers ran in 2 minutes with zero). + // A fresh child costs seconds (order file + env mmap). + served += 1; + if served > WORKER_RECYCLE_RANGES { + served = 1; + match pool.spawn(slot) { + Ok(w) => { + std::mem::replace(&mut worker, w).reap(); + fresh = true; + }, + Err(err) => { + let mut f = failure.lock().unwrap(); + if f.is_none() { + *f = Some(err); + } + abort.store(true, Ordering::Release); + break; + }, + } + } + match pool.scan(&mut worker, lo, hi) { + Ok(reply) => { + fresh = false; + let was_deferred = reply.deferred; + let Ok(next) = commit(reply, origin, seq) else { + abort.store(true, Ordering::Release); + break; + }; + if was_deferred && next < hi { + // Growth-threshold handoff: the walked prefix is banked; + // the dense remainder waits for the fat phase. + eprintln!( + "[scan] range {next}..{hi} defers to the fat phase \ + (growth threshold)" + ); + deferred.lock().unwrap().push((origin, next, hi)); + progress(hi.saturating_sub(lo)); + } else { + progress(next.saturating_sub(lo)); + requeue(origin, seq + 1, next, hi); + } + }, + Err(partial) => { + let was_fresh = fresh; + let e = match commit(partial, origin, seq) { + Ok(n) => n, + Err(()) => { + abort.store(true, Ordering::Release); + break; + }, + }; + progress(e.saturating_sub(lo)); + let status = std::mem::replace( + &mut worker, + match pool.spawn(slot) { + Ok(w) => w, + Err(err) => { + let mut f = failure.lock().unwrap(); + if f.is_none() { + *f = Some(err); + } + abort.store(true, Ordering::Release); + break; + }, + }, + ) + .reap(); + fresh = true; + if e >= hi { + eprintln!( + "[scan] worker {slot} died ({status}) after completing \ + its range; respawned" + ); + } else if e > lo || !was_fresh { + // A cache-shed kill: the respawned child (fresh record + // and decode cache) continues from the committed index. + // An AGED child dying on a range's first claim indicts + // its cache, not the block — the range requeues intact + // and waits for a fresh owner to judge it. + eprintln!( + "[scan] worker {slot} died ({status}) at index {e}; \ + respawned, continuing" + ); + requeue(origin, seq + 1, e, hi); + } else { + // A fresh worker died on the range's opening claim: this + // resume point cannot even open under a slot. In the + // fleet, the WHOLE remainder defers to the cleanup round, + // which walks it cumulatively under a fat cap — deferring + // only the block would re-pay the strip's cone per block, + // one doomed execution each (measured: 1,086 deferrals). + // In the cleanup round itself the block is named + // resource-infeasible and the walk continues past it. + let addr = pool.blocks[pool.order[e] as usize].addr.clone(); + if pool.cleanup { + eprintln!( + "[scan] block {} exceeded the cleanup cap ({status}) \ + — resource-infeasible; skipped", + addr.hex() + ); + failed.lock().unwrap().push(( + addr, + format!( + "record outgrew the {:.1} GiB cleanup cap mid-claim \ + (cgroup OOM-kill)", + f64_from_usize( + usize::try_from(pool.cap_bytes).unwrap_or(usize::MAX) + ) / GIB + ), + )); + progress(1); + requeue(origin, seq + 1, e + 1, hi); + } else { + eprintln!( + "[scan] range {e}..{hi} cannot open under its slot \ + cap ({status}); deferred to the cleanup round" + ); + let _ = addr; + deferred.lock().unwrap().push((origin, e, hi)); + progress(hi - e); + } + } + }, + } + in_flight.fetch_sub(1, Ordering::AcqRel); + } + worker.reap(); + }); + } + }); + if let Some(e) = failure.into_inner().unwrap() { + return Err(e); + } + let mut tagged = done.into_inner().unwrap(); + tagged.sort_by_key(|(k, _)| *k); + let mut segments: Vec = Vec::new(); + for (_, mut segs) in tagged { + segments.append(&mut segs); + } + let mut deferred = deferred.into_inner().unwrap(); + // Defer-infeasible mode: name every deferred block + // resource-infeasible instead of walking the deferred ranges under + // fat caps. The deferred region's cost is cone-bound kernel + // execution (measured ~6-10 worker-hours on FLT's typeclass-instance + // core in every slot configuration), so a caller can choose a + // partition of the tractable content NOW plus an exact exclusion + // inventory, over an hours-long exhaustive walk. + if !deferred.is_empty() && pool.defer_infeasible { + deferred.sort_unstable(); + deferred.dedup(); + let mut f = failed.lock().unwrap(); + let mut named = 0usize; + for &(_, lo, hi) in &deferred { + for &b in &pool.order[lo..hi] { + f.push(( + pool.blocks[b as usize].addr.clone(), + "deferred dense-core block (IX_SCAN_DEFER_INFEASIBLE=1): \ + opening cone exceeds a fleet slot; not measured" + .to_string(), + )); + named += 1; + } + } + eprintln!( + "[scan] defer-infeasible: {named} deferred block(s) in \ + {} range(s) named infeasible without measurement", + deferred.len() + ); + return Ok(segments); + } + if !deferred.is_empty() { + // Cleanup round: the deferred ranges re-run through this same + // function under fat caps — the drained pool split a few ways — so + // dense strips WALK (cone shared, segments cut gracefully at the + // fat soft cut) instead of being excluded by a slot's even share. + // Sorted for a deterministic round; `cleanup: true` names blocks + // that still cannot open resource-infeasible. + deferred.sort_unstable(); + deferred.dedup(); + // The fat phase's soft cut is cap-derived: dense-region cones run + // 12-16+ GiB, so segments must exceed the cone to amortize it — a + // small quantum re-pays the cone per segment (measured: no faster + // and many more false infeasibles). + let cleanup_soft_gib = (f64_from_usize( + usize::try_from(pool.cleanup_cap_bytes).unwrap_or(usize::MAX), + ) / GIB + - WORKER_BASELINE_GIB + - CLAIM_HEADROOM_GIB) + .max(1.0); + let cleanup_pool = ProcPool { + bin: pool.bin.clone(), + ixe: pool.ixe.clone(), + cut_used_gib: pool.cut_used_gib, + batch_blocks: pool.batch_blocks, + soft_record_gib: cleanup_soft_gib, + pieces: pool.pieces, + exec_only: pool.exec_only, + cap_bytes: pool.cleanup_cap_bytes, + cleanup_cap_bytes: pool.cleanup_cap_bytes, + cleanup: true, + order_file: pool.order_file.clone(), + capped: pool.capped, + order: pool.order, + blocks: pool.blocks, + fail_fast: pool.fail_fast, + defer_infeasible: false, + defer_growth: false, + }; + // Coalesce adjacent deferred ranges: strip remainders abut when a + // strip spans chunk edges, and a merged range shares its dependency + // cone across the walk. + let mut ranges: Vec<(usize, usize)> = Vec::new(); + for &(_, lo, hi) in &deferred { + match ranges.last_mut() { + Some((_, top)) if *top >= lo => *top = (*top).max(hi), + _ => ranges.push((lo, hi)), + } + } + let total: usize = ranges.iter().map(|(lo, hi)| hi - lo).sum(); + eprintln!( + "[scan] cleanup round: {total} deferred block(s) in {} range(s), {} \ + workers × {:.1} GiB", + ranges.len(), + CLEANUP_WORKERS.min(ranges.len()), + f64_from_usize( + usize::try_from(pool.cleanup_cap_bytes).unwrap_or(usize::MAX) + ) / GIB + ); + let extra = run_pool_procs( + &cleanup_pool, + ranges, + CLEANUP_WORKERS.min(deferred.len()), + failed, + )?; + segments.extend(extra); + // Deferred singles landed out of order; restore schedule adjacency + // so the merge pass sums true neighbors. + let pos: std::collections::HashMap = + pool.order.iter().enumerate().map(|(i, &b)| (b, i)).collect(); + segments.sort_by_key(|s| pos.get(&s.blocks[0]).copied().unwrap_or(0)); + } + Ok(segments) +} + +/// Execute-only mode: run the whole env's check schedule through the +/// codegen'd kernel in parallel — no partition, no manifest, no prove +/// concerns. Segments exist only to drop records (cut when a worker's +/// record bytes reach its planned share of box RAM), and the report is +/// the check verdict: blocks checked, kernel rejects named, total +/// measured FFT cost. This is the Aiur-kernel counterpart of the Rust +/// kernel's whole-env check, for wall-clock comparison and for finding +/// divergences (constants one kernel accepts and the other rejects). +pub fn execute_env( + toplevel: &Toplevel, + fun_idx: usize, + env: &Arc, + workers: usize, + fail_fast: bool, + proc_workers: Option<(&str, &str)>, +) -> Result { + let (blocks, adj) = schedule_blocks(env); + if blocks.is_empty() { + return Err("empty environment".to_string()); + } + let env_bytes: u64 = blocks.iter().map(|b| b.size).sum(); + let workers = if workers == 0 { + std::thread::available_parallelism() + .map_or(4, usize::from) + .saturating_sub(2) + .max(1) + } else { + workers + }; + // Provisional: re-bounded by pool/cap-floor once the measured + // baseline is known (proc mode only). + + let batch_blocks = std::env::var("IX_SCAN_BATCH_BLOCKS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&k| k >= 1) + .unwrap_or(SCAN_BATCH_BLOCKS); + // The schedule granularity is fixed by the core count so worker sizing + // cannot change the partition. + let sched_pieces = + (std::thread::available_parallelism().map_or(4, usize::from) * 2) + .min(blocks.len()) + .max(16); + let order = ordered_schedule(&blocks, &adj, sched_pieces); + let covered = order.len(); + // Post-schedule RSS: the parent's decode cache and static structures — + // the residency the worker fleet's record budget sits on top of. + let baseline_gib = process_rss_gib(); + // `IX_SCAN_RAM_GIB` overrides detected box RAM: every derived number + // (pool, width, caps, fat-phase slots) then scales exactly as a box + // of that size would — a budget emulation knob for capacity tests. + let ram = std::env::var("IX_SCAN_RAM_GIB") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| crate::kernel::system_ram_gib().unwrap_or(64.0)); + // Fleet bound by construction: Σ worker slices + parent + OS reserve + + // the env's page-cache residency (×2 re-read slack) = box RAM. Width + // goes to the core count when the pool affords a floor slice per + // worker. + let env_cache_gib = 2.0 * f64_from_usize( + usize::try_from(env_bytes).unwrap_or(usize::MAX), + ) / GIB; + let pool_gib = ((ram - baseline_gib - OS_RESERVE_GIB - env_cache_gib) + * POOL_SLICE_FRAC) + .max(SLICE_FLOOR_GIB); + let workers = + bound_workers_by_pool(workers, pool_gib, proc_workers.is_some()); + // Record drop: each worker's even slice of the pool — segments cut + // (and drop their record) when the record's exact heap reaches it. + let record_cut_gib = (pool_gib / f64_from_usize(workers)) + - WORKER_BASELINE_GIB + - CLAIM_HEADROOM_GIB; + let record_cut_gib = record_cut_gib.max(1.0); + let n_chunks = (blocks.len() / CHUNK_TARGET_BLOCKS) + .max(workers * 2) + .min(blocks.len()); + let bounds = make_chunk_bounds(&order, &blocks, env_bytes, n_chunks); + eprintln!( + "[exec] {} blocks, {workers} workers over {} chunks, record \ + drop at {record_cut_gib:.1} GiB, {batch_blocks} blocks per claim", + blocks.len(), + bounds.len() + ); + let failed: Mutex> = Mutex::new(Vec::new()); + let abort = std::sync::atomic::AtomicBool::new(false); + let ctx = ScanCtx { + toplevel, + fun_idx, + env, + blocks: &blocks, + cut_used_gib: record_cut_gib, + n_chunks: bounds.len(), + fail_fast, + failed: &failed, + abort: &abort, + batch_blocks, + system: None, + defer_growth: Some(DEFER_GROWTH_BYTES_PER_BLOCK), + soft_record_gib: f64::INFINITY, + }; + let segments = match proc_workers { + Some((bin, ixe)) => { + let cap_gib = pool_gib / f64_from_usize(workers); + let soft_gib = record_cut_gib; + let capped = systemd_scope_caps_available(); + if !capped { + eprintln!( + "[exec] systemd-run --user scopes unavailable — workers run \ + UNCAPPED" + ); + } + eprintln!( + "[exec] process pool: {workers} workers, {cap_gib:.1} GiB cap \ + each{}", + if capped { " (systemd MemoryMax)" } else { "" } + ); + let order_file = std::env::temp_dir() + .join(format!("ix-scan-order-{}.bin", std::process::id())); + write_order_file(&order_file, &blocks, &order)?; + let pool = ProcPool { + bin: bin.to_string(), + ixe: ixe.to_string(), + cut_used_gib: soft_gib, + batch_blocks, + soft_record_gib: soft_gib, + pieces: sched_pieces, + exec_only: true, + cap_bytes: gib_to_bytes_u64(cap_gib), + cleanup_cap_bytes: gib_to_bytes_u64( + pool_gib / f64_from_usize(CLEANUP_WORKERS), + ), + cleanup: false, + order_file, + capped, + order: &order, + blocks: &blocks, + fail_fast, + defer_infeasible: false, + defer_growth: true, + }; + run_pool_procs(&pool, bounds, workers, &failed)? + }, + None => { + let chunks = + bounds.iter().map(|&(lo, hi)| order[lo..hi].to_vec()).collect(); + run_pool(&ctx, chunks, workers)? + }, + }; + let total_fft: f64 = segments.iter().map(|s| s.fft).sum(); + let checked: usize = segments.iter().map(|s| s.blocks.len()).sum(); + let failed = failed.into_inner().unwrap(); + let mut report = format!( + "execute: {checked}/{covered} blocks checked in {} segment(s), total \ + measured {:.1} BFFT", + segments.len(), + total_fft / 1e9, + ); + if !failed.is_empty() { + report.push_str(&format!( + "\n [{} kernel-rejected block(s) SKIPPED:]", + failed.len() + )); + for (a, e) in &failed { + report.push_str(&format!("\n {} — {e}", a.hex())); + } + } + Ok(report) +} + +/// Scan-and-cut over the whole env: returns the manifest report, writing +/// the manifest and its costs sidecar to `out_path`. +#[allow(clippy::too_many_arguments)] +/// The predicted-vs-measured margin the cut leaves under the budget: +/// the analytic model predicts live bytes, and measured MaxRSS runs a +/// few percent above (allocator slack, the prove process's own env +/// decode cache, OS overhead) — validated at +3.0% worst across a +/// stratified Init prove sample. 0.95 covers it with room. +const PROVE_RAM_MARGIN: f64 = 0.95; + +/// GiB → whole bytes via the decimal round-trip (no `as` cast); caps are +/// small positive magnitudes. +fn gib_to_bytes_u64(gib: f64) -> u64 { + format!("{:.0}", (gib * GIB).max(0.0)).parse().unwrap_or(u64::MAX) +} + +pub fn scan_shards( + system: &AiurSystem, + fun_idx: usize, + env: &Arc, + budget_gib: f64, + eps: f64, + workers: usize, + fail_fast: bool, + defer_infeasible: bool, + out_path: &str, + proc_workers: Option<(&str, &str)>, +) -> Result { + let toplevel = system.toplevel(); + if budget_gib < 4.0 { + return Err(format!( + "budget {budget_gib} GiB is below the prover's fixed floor \ + (preprocessed gadget tables + base structures)" + )); + } + // The ε-discounted cut: a shard ends when its predicted peak prove + // RSS (analytic, from circuit shapes) reaches the margined budget. + let cut_used_gib = budget_gib * PROVE_RAM_MARGIN * (1.0 - eps); + + let (blocks, adj) = schedule_blocks(env); + if blocks.is_empty() { + return Err("empty environment".to_string()); + } + let env_bytes: u64 = blocks.iter().map(|b| b.size).sum(); + let workers = if workers == 0 { + std::thread::available_parallelism() + .map_or(4, usize::from) + .saturating_sub(2) + .max(1) + } else { + workers + }; + // Provisional: re-bounded by pool/cap-floor once the measured + // baseline is known (proc mode only). + + let batch_blocks = std::env::var("IX_SCAN_BATCH_BLOCKS") + .ok() + .and_then(|v| v.parse::().ok()) + .filter(|&k| k >= 1) + .unwrap_or(SCAN_BATCH_BLOCKS); + // The schedule granularity is fixed by the core count so worker sizing + // cannot change the partition. + let sched_pieces = + (std::thread::available_parallelism().map_or(4, usize::from) * 2) + .min(blocks.len()) + .max(16); + let order = ordered_schedule(&blocks, &adj, sched_pieces); + // Post-schedule RSS: the parent's decode cache and static structures — + // the residency the worker pool's budget sits on top of. + let baseline_gib = process_rss_gib(); + // Width-first sizing: full core width while every worker's even slice + // of the pool clears the floor. Claim widths derived from measured + // growth bound mid-claim overshoot on any content, so slices are + // segment quanta, not worst-case-claim reserves — the merge pass sums + // segments to the cut, and the kernel kill stays a backstop. Blocks + // too heavy even for a slice are deferred to the fat-cap cleanup + // round or named infeasible. + // `IX_SCAN_RAM_GIB` overrides detected box RAM: every derived number + // (pool, width, caps, fat-phase slots) then scales exactly as a box + // of that size would — a budget emulation knob for capacity tests. + let ram = std::env::var("IX_SCAN_RAM_GIB") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| crate::kernel::system_ram_gib().unwrap_or(64.0)); + // Fleet bound by construction: Σ worker slices + parent + OS reserve + + // the env's page-cache residency = box RAM. The env term keeps the + // shared mmap cache-resident (×2 for re-read slack): without it, a + // fleet at its caps evicts the very pages every worker faults from. + let env_cache_gib = 2.0 * f64_from_usize( + usize::try_from(env_bytes).unwrap_or(usize::MAX), + ) / GIB; + let pool_gib = ((ram - baseline_gib - OS_RESERVE_GIB - env_cache_gib) + * POOL_SLICE_FRAC) + .max(SLICE_FLOOR_GIB); + let workers = + bound_workers_by_pool(workers, pool_gib, proc_workers.is_some()); + let proc_cap_gib = proc_workers.map(|_| { + std::env::var("IX_SCAN_WORKER_CAP_GIB") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| pool_gib / f64_from_usize(workers)) + }); + let n_chunks = (blocks.len() / CHUNK_TARGET_BLOCKS) + .max(workers * 2) + .min(blocks.len()); + let bounds = make_chunk_bounds(&order, &blocks, env_bytes, n_chunks); + let chunk_count = bounds.len(); + eprintln!( + "[scan] {} blocks, {workers} workers over {chunk_count} chunks, cut at \ + {cut_used_gib:.1} GiB predicted prove RSS (margin \ + {:.0}%, ε pre-charged), {batch_blocks} blocks per claim", + blocks.len(), + (1.0 - PROVE_RAM_MARGIN) * 100.0 + ); + let failed: Mutex> = Mutex::new(Vec::new()); + let abort = std::sync::atomic::AtomicBool::new(false); + let ctx = ScanCtx { + toplevel, + fun_idx, + env, + blocks: &blocks, + cut_used_gib, + n_chunks: chunk_count, + fail_fast, + failed: &failed, + abort: &abort, + batch_blocks, + system: Some(system), + defer_growth: None, + soft_record_gib: f64::INFINITY, + }; + let segments = match proc_workers { + Some((bin, ixe)) => { + let cap_gib = proc_cap_gib.unwrap_or(16.0); + let soft_gib = + (cap_gib - WORKER_BASELINE_GIB - CLAIM_HEADROOM_GIB).max(1.0); + let order_file = std::env::temp_dir() + .join(format!("ix-scan-order-{}.bin", std::process::id())); + write_order_file(&order_file, &blocks, &order)?; + let capped = systemd_scope_caps_available(); + if !capped { + eprintln!( + "[scan] systemd-run --user scopes unavailable — workers run \ + UNCAPPED" + ); + } + eprintln!( + "[scan] process pool: {workers} workers, {cap_gib:.1} GiB cap \ + each{}, soft record cut {soft_gib:.1} GiB", + if capped { " (systemd MemoryMax)" } else { "" }, + ); + let pool = ProcPool { + bin: bin.to_string(), + ixe: ixe.to_string(), + cut_used_gib, + batch_blocks, + soft_record_gib: soft_gib, + pieces: sched_pieces, + exec_only: false, + cap_bytes: gib_to_bytes_u64(cap_gib), + cleanup_cap_bytes: gib_to_bytes_u64( + pool_gib / f64_from_usize(CLEANUP_WORKERS), + ), + cleanup: false, + order_file, + capped, + order: &order, + blocks: &blocks, + fail_fast, + defer_infeasible, + defer_growth: false, + }; + run_pool_procs(&pool, bounds, workers, &failed)? + }, + None => { + let chunks = + bounds.iter().map(|&(lo, hi)| order[lo..hi].to_vec()).collect(); + run_pool(&ctx, chunks, workers)? + }, + }; + + // Assemble shards by UNION pricing adjacent segments. A candidate + // shard's true cold cost is the cost of the UNION of its segments' + // records — per-query cost is context-free, so shared cone queries + // deduplicate exactly as they will in the prove's single cold record. + // The merge unions the segments' membership sketches, feeds the + // estimated union heights to the same analytic cost/RAM model the cut + // charges, and closes a shard where the modeled union RAM reaches the + // cut — no overshoot pack, no cold re-price pass. Summed costs + // overstated true cost by a measured 1.3-1.8x (cross-segment cone + // double-counting); the union estimate is exact where every map + // stayed under [`SKETCH_EXACT_MAX`] uniques and within ~±2% (erring + // conservative) where HLL registers took over. + let pre_merge = segments.len(); + let mut packed: Vec<(Vec, f64, f64)> = Vec::new(); + { + let mut cur_blocks: Vec = Vec::new(); + let mut acc = UnionAcc::default(); + let mut cur_cost = (0.0f64, 0.0f64); + for seg in segments { + let mut trial = acc.clone(); + trial.absorb(&seg); + let (tf, tr) = trial.model(system); + if tr >= cut_used_gib && !cur_blocks.is_empty() { + // Close the running shard; this segment opens the next. + packed.push(( + std::mem::take(&mut cur_blocks), + cur_cost.0, + cur_cost.1, + )); + acc = UnionAcc::default(); + acc.absorb(&seg); + cur_cost = acc.model(system); + } else { + acc = trial; + cur_cost = (tf, tr); + } + cur_blocks.extend(&seg.blocks); + } + if !cur_blocks.is_empty() { + packed.push((cur_blocks, cur_cost.0, cur_cost.1)); + } + } + let segments: Vec = packed + .into_iter() + .map(|(blocks, fft, ram_gib)| Segment { + blocks, + fft, + ram_gib, + sketch: None, + }) + .collect(); + + // Manifest: owned blocks per segment; frontier fields are the claim + // layer's business (reconstructed from env + owned at check/prove time). + let num = segments.len(); + let mut infos = Vec::with_capacity(num); + for (id, seg) in segments.iter().enumerate() { + let mut addrs: Vec
= + seg.blocks.iter().map(|&b| blocks[b as usize].addr.clone()).collect(); + addrs.sort(); + let own_size: u64 = + seg.blocks.iter().map(|&b| blocks[b as usize].size).sum(); + infos.push(ShardInfo { + id: u32::try_from(id).expect("shard count exceeds u32"), + blocks: addrs, + cost: ShardCost::AiurFft(cost_fft(seg.fft)), + own_size, + foreign_blocks: Vec::new(), + cross_ingress: 0, + assumption_root: None, + }); + } + let num_u32 = u32::try_from(num).expect("shard count exceeds u32"); + let manifest = ShardManifest { + num_shards: num_u32, + shards: infos, + total_cross_ingress: 0, + tree: Some(balanced_agg_tree(0, num_u32)), + }; + std::fs::write(out_path, manifest.to_bytes()) + .map_err(|e| format!("write {out_path}: {e}"))?; + + // Costs sidecar: MEASURED fft per shard mapped through the calibrated + // resource lines — same header the batch prove driver's heaviest-first + // ordering reads; the counter columns are zero (nothing was predicted). + let mut csv = String::from( + "shard,union_bytes,hb,subst,subst_unique,whnf,def_eq,nat_arith,\ + pred_ram_gib,pred_prove_s\n", + ); + let mut max_ram = 0.0f64; + let mut over = 0usize; + for (id, seg) in segments.iter().enumerate() { + let own: u64 = seg.blocks.iter().map(|&b| blocks[b as usize].size).sum(); + // Predicted prove RSS = the fft resource line PLUS the measured + // record bytes the prove's execute replays into — the second term is + // what the fitted line missed on arithmetic-heavy shards. + let ram = seg.ram_gib; + csv.push_str(&format!( + "{},{},0,0,0,0,0,0,{:.2},{:.2}\n", + id, + own, + ram, + aiur_prove_secs_for_fft(seg.fft), + )); + max_ram = max_ram.max(ram); + if seg.ram_gib >= cut_used_gib / (1.0 - eps) { + over += 1; + } + } + let cp = format!("{out_path}.costs.csv"); + std::fs::write(&cp, csv).map_err(|e| format!("write {cp}: {e}"))?; + + let mut note = if over > 0 { + format!( + "\n [{over} single-block segment(s) exceed the cap alone — atomically \ + infeasible at this budget]" + ) + } else { + String::new() + }; + let failed = failed.into_inner().unwrap(); + if !failed.is_empty() { + let mut fcsv = String::from("block,error\n"); + for (a, e) in &failed { + fcsv.push_str(&format!( + "{},{}\n", + a.hex(), + e.replace('\n', " ").replace(',', ";") + )); + } + let fp = format!("{out_path}.failed.csv"); + std::fs::write(&fp, fcsv).map_err(|e| format!("write {fp}: {e}"))?; + note.push_str(&format!( + "\n [{} kernel-rejected block(s) SKIPPED — the partition does NOT \ + cover them (the coverage gate will name them); see {fp}]", + failed.len() + )); + } + Ok(format!( + "scan: {} blocks in {} chunks → {num} shards ({pre_merge} pre-merge) @ \ + {budget_gib:.0} GiB (cut at {:.1} GiB combined, ε {:.0}%)\nmax \ + predicted prove RSS {max_ram:.1} GiB (analytic, from circuit \ + shapes){note}", + blocks.len(), + chunk_count, + cut_used_gib, + eps * 100.0, + )) +} + +/// Number of segments one work-range yields before its remainder goes +/// back on the queue for any idle worker. +const RANGE_SEGMENTS: usize = 2; + +/// Scan one range: execute thin-frontier `CheckEnv` claims — one per +/// BATCH of [`ScanCtx::batch_blocks`] blocks — against a shared record +/// and lazily-faulted witness, checkpointing the running (fft, record +/// bytes) after every claim and cutting on the batch boundary where it +/// reaches the cut (the crossing batch re-executes as the next segment's +/// first claim, so an emitted shard never exceeds the cut). Batching is +/// what keeps the running readout honest: the claim layer's per-claim +/// costs (in-circuit assumption-tree hashing, `env_walk` frames that are +/// never memo-shared across claims, members assumed by one claim then +/// checked by the next) shrink ~K-fold, and intra-batch edges stop being +/// frontier members entirely, so the checkpoint stays a tight upper +/// bound on the emitted shard's cold cost without a blanket re-price. +/// +/// Any batch-level event that needs per-block attribution — the segment's +/// FIRST claim crossing the cut, a kernel reject, or a record-cap trip +/// with nothing banked — ends the segment at the last clean checkpoint +/// (the polluted record is dropped) and re-enters that batch through a +/// NARROW window, one block per claim, where the single-block semantics +/// apply verbatim: a lone block over the cut is emitted alone with its +/// measured cost, a rejected or over-cap block is named and skipped. +/// Emits at most [`RANGE_SEGMENTS`] segments, then returns the remaining +/// blocks for any idle worker; a remainder re-queued mid-window simply +/// rediscovers the event deterministically. +fn scan_range( + ctx: &ScanCtx<'_>, + chunk: &[u32], + origin: u32, +) -> Result<(Vec, Vec, bool), String> { + let t0 = std::time::Instant::now(); + let chunk_id = origin; + let n_chunks = ctx.n_chunks; + let mut segments: Vec = Vec::new(); + let mut lo = 0usize; + // Set when measured growth crossed the phase-1 threshold: the range's + // remainder is handed back marked for the fat phase. + let mut defer_rest = false; + // Blocks below this index (and at/after `lo`) execute one per claim: + // a batch-level event landed in [lo, narrow_until) and needs per-block + // attribution. Stale values (< hi) are inert. + let mut narrow_until = 0usize; + // Running record growth per block (bytes), from the last claim's + // measured growth — the range's execution history is content-fixed, + // so the estimate (and thus every claim width) is deterministic. + // `None` until the first claim measures. + let mut growth_per_block: Option = None; + while lo < chunk.len() && segments.len() < RANGE_SEGMENTS && !defer_rest { + let mut record = QueryRecord::new(ctx.toplevel); + let mut io = IOBuffer::with_backing(EnvFaultSource::new(ctx.env.clone())); + let mut prev_fft = 0.0f64; + let mut prev_ram = 0.0f64; + let mut hi = lo; + let mut skip_failed = false; + let (seg_end, seg_fft, seg_ram) = loop { + if hi >= chunk.len() { + break (hi, prev_fft, prev_ram); + } + if ctx.abort.load(Ordering::Acquire) { + return Err("aborted after a failure elsewhere".to_string()); + } + // Claim width from measured growth: K sized so this claim's + // expected record growth is ~CLAIM_TARGET_GIB. Light content runs + // full width; dense content shrinks to K=1-2, where per-claim + // overhead is negligible against per-block cost — one rule bounds + // mid-claim growth on every content class. Before the first + // measurement the estimate is unknown and the claim starts small. + let k = if hi < narrow_until { + 1 + } else { + let by_growth = match growth_per_block { + // No measurement yet: a single block seeds the estimator. A + // wider opening claim cascades in monster strips — after a + // deferral the follow-up range also starts unmeasured, so any + // multi-block seed re-dies block after block (measured: 998 + // one-death-one-deferral blocks at a 4-block seed, where the + // same strips scanned clean once estimators were trained). + None => 1, + Some(g) => { + let target = CLAIM_TARGET_GIB * GIB; + format!("{:.0}", (target / g.max(1.0)).clamp(1.0, 4096.0)) + .parse::() + .unwrap_or(1) + .min(ctx.batch_blocks) + }, + }; + by_growth.min(chunk.len() - hi) + }; + let addrs: Vec
= chunk[hi..hi + k] + .iter() + .map(|&b| ctx.blocks[b as usize].addr.clone()) + .collect(); + let heap_before = record_heap_bytes(&record); + let out: Result<(), String> = seed_shard_check_env_claim( + ctx.env, &addrs, &mut io, + ) + .and_then(|(_claim, input)| { + execute_ixvm_with_record( + ctx.toplevel, + ctx.fun_idx, + &input, + &mut io, + &mut record, + ) + .map(|_| ()) + .map_err(|e| e.to_string()) + }); + growth_per_block = Some( + (f64_from_usize(record_heap_bytes(&record).saturating_sub( + heap_before, + )) / f64_from_usize(k)) + .max(1.0), + ); + // Phase-1 growth deferral: the claim that just executed is banked + // (its checks are done), and the remainder waits for the fat + // phase — dense content is only cheap walked warm with room, and + // the threshold sits in the decade-wide gap between the light and + // dense growth populations. + if let (Some(threshold), Some(g)) = (ctx.defer_growth, growth_per_block) + && g > threshold + && hi >= narrow_until + { + defer_rest = true; + break (hi + k, match ctx.system { + Some(sys) => sys.fft_cost_of_record(&record), + None => record_fft_cost(ctx.toplevel, &record), + }, { + let rec = + f64_from_usize(record_heap_bytes(&record)) / GIB; + match ctx.system { + Some(sys) => { + f64_from_usize(sys.peak_prove_bytes(&record).peak) / GIB + }, + None => rec, + } + }); + } + if let Err(e) = out { + if k > 1 { + // Per-block attribution needed: end the segment at the last + // clean checkpoint (dropping the polluted record) and rescan + // this batch one block per claim. + eprintln!( + "[scan {chunk_id}/{n_chunks}] narrowing batch at block \ + {hi}: {e}" + ); + narrow_until = hi + k; + break (hi, prev_fft, prev_ram); + } + let addr = &addrs[0]; + if ctx.fail_fast { + return Err(format!( + "CheckEnv of block {} failed during scan: {e} \ + (--no-fail-fast records and skips such blocks)", + addr.hex() + )); + } + // No fail-fast: name the block NOW, drop it from the partition, + // and emit the running segment so the failed execution's partial + // rows cannot pollute later measurements (fresh record). + eprintln!( + "[scan {chunk_id}/{n_chunks}] SKIPPING block {}: {e}", + addr.hex() + ); + ctx.failed.lock().unwrap().push((addr.clone(), e)); + skip_failed = true; + break (hi, prev_fft, prev_ram); + } + // Scan-mode fft is the 539-grounded model over the record's raw + // heights — one unit across manifest, whole-env, and per-constant + // figures; execute-only keeps the legacy readout (display only). + let fft = match ctx.system { + Some(sys) => sys.fft_cost_of_record(&record), + None => record_fft_cost(ctx.toplevel, &record), + }; + let rec_gib = f64_from_usize(record_heap_bytes(&record)) / GIB; + // The cut measure: for the scan, the analytic peak-prove-RSS + // prediction from the record's circuit shapes; for execute-only + // segments (which exist only to bound the live record), the + // record's retained bytes against the per-worker share. + let ram_gib = match ctx.system { + Some(sys) => f64_from_usize(sys.peak_prove_bytes(&record).peak) / GIB, + None => rec_gib, + }; + if ram_gib >= ctx.cut_used_gib || rec_gib >= ctx.soft_record_gib { + if hi == lo { + if k > 1 { + // The segment's first batch crosses the whole cut: find the + // culprit at per-block granularity before emitting anything. + eprintln!( + "[scan {chunk_id}/{n_chunks}] narrowing batch at block \ + {hi}: first claim crossed the cut" + ); + narrow_until = hi + k; + break (hi, prev_fft, prev_ram); + } + // A single block alone reaches the cut: atomically infeasible + // at this budget — emitted alone with its measured cost. + break (hi + 1, fft, ram_gib); + } + break (hi, prev_fft, prev_ram); + } + if segments.is_empty() && lo == 0 && hi == lo { + // Bank the range's very first claim as its own segment: the + // committed index then advances past it immediately, so any + // later death in this range preserves progress (`e > lo`) and + // is answered by respawn-and-continue. A zero-progress death + // therefore means exactly "this claim, alone, on this worker" — + // the conviction the deferral path assumes. One extra seed-size + // segment per range; the merge pass absorbs it. + break (hi + k, fft, ram_gib); + } + hi += k; + prev_fft = fft; + prev_ram = ram_gib; + }; + if query_stats_enabled() { + dump_query_stats(&record, &format!("scan {chunk_id} seg")); + } + // A failure on a segment's FIRST block leaves nothing to emit; the + // failed block itself is skipped either way (`skip_failed`). + if seg_end > lo { + // Memory decomposition per segment: witness arena G-elements, io + // map entries, record entries — with process RSS, these split the + // footprint into decode-cache baseline / live worker data / + // unaccounted, so a single run localizes any growth. + let arena_g: usize = io.data.values().map(Vec::len).sum(); + let rec_e: usize = + record.function_queries.iter().map(|m| m.len()).sum::() + + record.memory_queries.iter().map(|(_, m)| m.len()).sum::(); + eprintln!( + "[scan {chunk_id}/{n_chunks}] segment: {} blocks, {:.2} BFFT, \ + {:.1}G {}, {}/{} blocks done, {:.0}s, rss {:.0}G, arena {}M, \ + iomap {}k, rec {}M entries/{:.1}G heap", + seg_end - lo, + seg_fft / 1e9, + seg_ram, + if ctx.system.is_some() { "pred-RSS" } else { "rec" }, + seg_end, + chunk.len(), + t0.elapsed().as_secs_f64(), + process_rss_gib(), + arena_g / 1_000_000, + io.map.len() / 1000, + rec_e / 1_000_000, + f64_from_usize(record_heap_bytes(&record)) / GIB + ); + segments.push(Segment { + blocks: chunk[lo..seg_end].to_vec(), + fft: seg_fft, + ram_gib: seg_ram, + // The record may hold a few rows past the checkpoint (a narrowed + // or skipped batch's partial execution) — extra members only + // raise the union estimate, the safe direction. + sketch: ctx.system.map(|_| SegSketch::of_record(&record)), + }); + } + lo = seg_end + usize::from(skip_failed); + } + if lo >= chunk.len() { + eprintln!( + "[scan {chunk_id}/{n_chunks}] range done: {} blocks → {} segment(s), \ + {:.0}s", + chunk.len(), + segments.len(), + t0.elapsed().as_secs_f64() + ); + } + Ok((segments, chunk[lo..].to_vec(), defer_rest)) +} + +use lean_ffi::object::{ + LeanBorrowed, LeanExcept, LeanExternal, LeanNat, LeanOwned, LeanString, +}; + +use crate::aiur::toplevel::decode_toplevel; +use crate::lean::LeanAiurToplevel; + +/// `Bytecode.Toplevel.scanShardsWithEnv`: scan-and-cut sharding against a +/// Rust-owned `EnvHandle`. Numeric params are decimal strings (ABI-simple): +/// `budget_gib` (RAM budget per shard, GiB), `eps_pct` (pre-charged cut +/// headroom, percent), `workers` (parallel chunk scanners; `0` autoscales +/// to cores and detected RAM — each worker holds one segment's QueryRecord +/// and faulted witness). Writes `out_path` (.ixes) plus its `.costs.csv` +/// sidecar carrying the MEASURED per-shard FFT mapped through the +/// calibrated resource lines. +#[unsafe(no_mangle)] +extern "C" fn rs_aiur_scan_shards_with_env( + system: LeanExternal>, + fun_idx: LeanNat>, + env_handle: LeanExternal< + ixvm_codegen::env_handle::EnvHandle, + LeanBorrowed<'_>, + >, + budget_gib: LeanString>, + eps_pct: LeanString>, + workers: LeanString>, + fail_fast: LeanString>, + out_path: LeanString>, + worker_bin: LeanString>, + ixe_path: LeanString>, +) -> LeanExcept { + let fun_idx = crate::aiur::lean_unbox_nat_as_usize(fun_idx.inner()); + let budget = budget_gib.to_string().parse::().unwrap_or(0.0); + if budget <= 0.0 { + return LeanExcept::error_string("scan: pass a positive RAM budget (GiB)"); + } + let eps = eps_pct.to_string().parse::().unwrap_or(5.0) / 100.0; + let workers = workers.to_string().parse::().unwrap_or(0); + // `fail_fast` mode string: "1" abort on the first kernel reject, "0" + // record-and-skip, "2" record-and-skip + name deferred dense ranges + // infeasible. + let mode = fail_fast.to_string(); + let fail_fast = mode == "1"; + let defer_infeasible = mode == "2"; + let (bin, ixe) = (worker_bin.to_string(), ixe_path.to_string()); + let proc_workers = (!bin.is_empty() && !ixe.is_empty()) + .then_some((bin.as_str(), ixe.as_str())); + match scan_shards( + system.get(), + fun_idx, + &env_handle.get().env, + budget, + eps, + workers, + fail_fast, + defer_infeasible, + &out_path.to_string(), + proc_workers, + ) { + Ok(report) => { + eprintln!("[rs_scan]\n{report}"); + LeanExcept::ok(LeanOwned::box_usize(0)) + }, + Err(e) => LeanExcept::error_string(&format!("rs_aiur_scan_shards: {e}")), + } +} + +/// `Bytecode.Toplevel.executeEnvWithEnv`: execute-only whole-env check +/// through the codegen'd Aiur kernel — no partition, no manifest (see +/// [`execute_env`]). Numeric params are decimal strings (ABI-simple): +/// `workers` (`0` autoscales), `fail_fast` (`0` records and skips +/// kernel-rejected blocks instead of aborting). +#[unsafe(no_mangle)] +extern "C" fn rs_aiur_execute_env_with_env( + toplevel: LeanAiurToplevel>, + fun_idx: LeanNat>, + env_handle: LeanExternal< + ixvm_codegen::env_handle::EnvHandle, + LeanBorrowed<'_>, + >, + workers: LeanString>, + fail_fast: LeanString>, + worker_bin: LeanString>, + ixe_path: LeanString>, +) -> LeanExcept { + let toplevel = decode_toplevel(&toplevel); + let fun_idx = crate::aiur::lean_unbox_nat_as_usize(fun_idx.inner()); + let workers = workers.to_string().parse::().unwrap_or(0); + let fail_fast = fail_fast.to_string() != "0"; + let (bin, ixe) = (worker_bin.to_string(), ixe_path.to_string()); + let proc_workers = (!bin.is_empty() && !ixe.is_empty()) + .then_some((bin.as_str(), ixe.as_str())); + match execute_env( + &toplevel, + fun_idx, + &env_handle.get().env, + workers, + fail_fast, + proc_workers, + ) { + Ok(report) => { + eprintln!("[rs_exec]\n{report}"); + LeanExcept::ok(LeanOwned::box_usize(0)) + }, + Err(e) => LeanExcept::error_string(&format!("rs_aiur_execute_env: {e}")), + } +} + +/// `Aiur.AiurSystem.scanWorker`: the child side of the process pool — +/// runs [`scan_worker`]'s stdin/stdout loop until EOF. Numeric params are +/// decimal strings: cut (GiB), batch blocks, soft record cut (GiB), +/// schedule pieces (must match the parent's chunk count), exec-only +/// ("1" = record-bytes cut, no model). +#[unsafe(no_mangle)] +extern "C" fn rs_aiur_scan_worker( + system: LeanExternal>, + fun_idx: LeanNat>, + env_handle: LeanExternal< + ixvm_codegen::env_handle::EnvHandle, + LeanBorrowed<'_>, + >, + cut_gib: LeanString>, + batch: LeanString>, + soft_cap_gib: LeanString>, + pieces: LeanString>, + exec_only: LeanString>, +) -> LeanExcept { + let fun_idx = crate::aiur::lean_unbox_nat_as_usize(fun_idx.inner()); + let cut = cut_gib.to_string().parse::().unwrap_or(f64::INFINITY); + let batch = batch.to_string().parse::().unwrap_or(SCAN_BATCH_BLOCKS); + let soft = soft_cap_gib.to_string().parse::().unwrap_or(f64::INFINITY); + let pieces = pieces.to_string().parse::().unwrap_or(16); + // Mode string: "0" scan, "1" execute-only, "2" execute-only with + // growth-threshold deferral (phase 1 of the two-phase execute). + let mode = exec_only.to_string(); + let exec_only = mode == "1" || mode == "2"; + let defer_growth = mode == "2"; + match scan_worker( + system.get(), + fun_idx, + &env_handle.get().env, + cut, + batch, + soft, + pieces, + exec_only, + defer_growth, + ) { + Ok(()) => LeanExcept::ok(LeanOwned::box_usize(0)), + Err(e) => LeanExcept::error_string(&format!("rs_aiur_scan_worker: {e}")), + } +} diff --git a/crates/ffi/src/kernel.rs b/crates/ffi/src/kernel.rs index f548a1ba..63a05c8d 100644 --- a/crates/ffi/src/kernel.rs +++ b/crates/ffi/src/kernel.rs @@ -69,7 +69,9 @@ use ix_kernel::ingress::{ #[cfg(feature = "test-ffi")] use ix_kernel::ingress::{ixon_ingress, lean_ingress}; use ix_kernel::mode::{Anon, CheckDupLevelParams, KernelMode, Meta}; -use ix_kernel::profile::{BlockProfile, OpCounts, ProfileBuilder, ProfileSink}; +use ix_kernel::profile::{ + BlockEntry, BlockProfile, OpCounts, ProfileBuilder, ProfileSink, +}; use ix_kernel::tc::TypeChecker; use ixon::constant::ConstantInfo as IxonCI; #[cfg(feature = "test-ffi")] @@ -2240,20 +2242,478 @@ fn profile_block_size(env: &IxonEnv, block: &Address) -> u32 { // `steps as f64` is a display-only cast for `{:.2e}` formatting; precision loss // past 2⁵³ steps is irrelevant to a two-sig-fig estimate. #[allow(clippy::cast_precision_loss)] +/// Display name per profile block: the lexicographically-smallest named +/// member (constants resolve through `profile_block_of`, so a projection's +/// name lands on its block; a directly-named `Muts` block keeps its own +/// `…._mutual`-style name). Smallest-wins keeps the pick deterministic. +/// The anon env carries no names (`get_anon_mmap` stops before §4-6), so +/// the §5 entries are re-read from the file via the lazy index — cheap: +/// the lazy parser skips constant and metadata bodies. Any read/parse +/// failure just yields an empty map (the leaderboards fall back to +/// addresses); names are presentation, never worth failing the profile. +fn block_display_names( + env: &IxonEnv, + path: &str, +) -> FxHashMap { + let Ok(bytes) = std::fs::read(path) else { + return FxHashMap::default(); + }; + let Ok(index) = IxonEnv::parse_lazy_index(&bytes) else { + return FxHashMap::default(); + }; + let mut names: FxHashMap = FxHashMap::default(); + // Compiler-generated `Ix.<64-hex>.…` aux aliases sort before every human + // name; prefer any human name over them, then smallest-wins. + let is_aux = |s: &str| { + s.strip_prefix("Ix.").is_some_and(|r| { + r.len() > 65 + && r.as_bytes()[64] == b'.' + && r.as_bytes()[..64].iter().all(u8::is_ascii_hexdigit) + }) + }; + let rank = |s: &str| (is_aux(s), s.to_owned()); + for ln in &index.named { + let block = profile_block_of(env, &ln.addr); + let s = format!("{}", ln.name); + names + .entry(block) + .and_modify(|cur| { + if rank(&s) < rank(cur) { + *cur = s.clone(); + } + }) + .or_insert(s); + } + names +} + +/// Print the top-`n` blocks by one metric, largest first, with display +/// names (falling back to the block address) and the block's member count. +fn print_top_blocks( + title: &str, + n: usize, + profile: &BlockProfile, + names: &FxHashMap, + metric: &dyn Fn(&BlockEntry) -> u64, +) { + let blocks = profile.blocks(); + let mut idx: Vec = (0..blocks.len()).collect(); + idx.sort_by_key(|&i| std::cmp::Reverse(metric(&blocks[i]))); + eprintln!("\n top {} blocks by {title}", n.min(blocks.len())); + for (rank, &i) in idx.iter().take(n).enumerate() { + let b = &blocks[i]; + let name = names.get(&b.addr).cloned().unwrap_or_else(|| b.addr.hex()); + let members = if b.const_count > 1 { + format!(" [{} consts]", b.const_count) + } else { + String::new() + }; + eprintln!(" {:>2}. {:>14} {name}{members}", rank + 1, metric(b)); + } +} + +/// Block-level reference adjacency for a profiled env: for each block, the +/// sorted, deduped, self-edge-free set of blocks it references. Projection +/// and mutual-member edges are intra-block by construction, so per-constant +/// `refs` folded to home blocks carry every cross-block edge of +/// `Env::bfs_closure`. This is the graph persisted into the `.ixprof` +/// (reachability over it = a block's full dependency closure — what an Aiur +/// shard's witness ships) and the sweep's walk graph. +fn build_block_ref_adjacency( + env: &IxonEnv, + profile: &BlockProfile, +) -> Vec> { + let blocks = profile.blocks(); + let n = blocks.len(); + let id_of: FxHashMap<&Address, u32> = blocks + .iter() + .enumerate() + .filter_map(|(i, b)| u32::try_from(i).ok().map(|id| (&b.addr, id))) + .collect(); + // Pass 1: every constant's home block id. + let mut home: FxHashMap = FxHashMap::default(); + for entry in env.consts.iter() { + let (addr, lazy) = (entry.key(), entry.value()); + let Ok(c) = lazy.get() else { continue }; + let home_addr = match &c.info { + IxonCI::IPrj(p) => &p.block, + IxonCI::CPrj(p) => &p.block, + IxonCI::RPrj(p) => &p.block, + IxonCI::DPrj(p) => &p.block, + _ => addr, + }; + if let Some(&id) = id_of.get(home_addr) { + home.insert(addr.clone(), id); + } + } + // Pass 2: fold each constant's refs to home-block edges. + let mut adj: Vec> = vec![Vec::new(); n]; + for entry in env.consts.iter() { + let (addr, lazy) = (entry.key(), entry.value()); + let Ok(c) = lazy.get() else { continue }; + let Some(&hid) = home.get(addr) else { continue }; + for r in &c.refs { + if let Some(&rid) = home.get(r) + && rid != hid + { + adj[hid as usize].push(rid); + } + } + } + for row in &mut adj { + row.sort_unstable(); + row.dedup(); + } + adj +} + +/// One root's closure-aggregated features + model predictions, as produced by +/// [`profile_sweep`]. +struct SweepRow { + name: String, + closure_blocks: u32, + bytes: u64, + hb: u64, + subst: u64, + subst_unique: u64, + whnf: u64, + def_eq: u64, + nat_arith: u64, + exec_s: f64, + exec_gib: f64, + prove_s: f64, + prove_gib: f64, + /// Membership mask over the sweep's tracked expensive blocks (bit `j` set ⇔ + /// this closure contains tracked block `j`). + hot_mask: u64, +} + +/// Env-wide closure cost sweep: for every named constant (one representative +/// name per home block), walk its full reference closure over the env's +/// constant graph, sum the whole-env profile's per-block counters over the +/// members, and apply the Aiur execute/prove models. One kernel profile run +/// amortizes over every query — nothing here re-checks or re-serializes. +/// +/// The closure is REFERENCE reachability (`Constant.refs`, with projections +/// folded into their home blocks), matching `ix shard extract`'s membership — +/// not the `.ixprof` delta graph, which under-approximates it (a constant can +/// be referenced but never delta-unfolded). Summing whole-env per-block +/// counters over the membership equals a fresh per-closure profile because +/// isolate-mode counters are per-block context-free. +/// +/// Reports, beyond the per-root CSV: +/// - feasibility at `budget_gib`: counts + the cheapest prove-infeasible +/// closures (minimal reproducers of the RAM bottleneck); +/// - min-root per hot block: for each of the `top_m` most expensive blocks +/// (by marginal Aiur prove cost), the cheapest closure containing it; +/// - `reps_k` diversity picks: prove-feasible closures with maximally +/// distinct feature mixes (farthest-point sampling, seeded on the highest +/// nat-arith share — the model's blind-spot axis). +#[allow(clippy::cast_precision_loss, clippy::cast_possible_truncation)] +pub fn profile_sweep( + env_path: &str, + prof_path: &str, + csv_out: &str, + budget_gib: f64, + top_m: usize, + reps_k: usize, +) -> Result { + use ix_kernel::shard::{ + AIUR_RAM_USABLE_FRAC, aiur_block_prove_secs, aiur_exec_ram_gib, + aiur_exec_secs, aiur_prove_secs, aiur_ram_gib, + }; + use rayon::prelude::*; + + let t0 = Instant::now(); + let env = IxonEnv::get_anon_mmap(std::path::Path::new(env_path)) + .map_err(|e| format!("sweep: mmap+deserialize {env_path}: {e}"))?; + let prof_bytes = std::fs::read(prof_path) + .map_err(|e| format!("sweep: read {prof_path}: {e}"))?; + let mut profile = BlockProfile::from_bytes(&prof_bytes) + .map_err(|e| format!("sweep: parse {prof_path}: {e}"))?; + if !profile.has_ref_graph() { + let adj = build_block_ref_adjacency(&env, &profile); + profile.set_ref_graph(&adj); + } + let blocks = profile.blocks(); + let n = blocks.len(); + let id_of: FxHashMap<&Address, u32> = + blocks.iter().enumerate().map(|(i, b)| (&b.addr, i as u32)).collect(); + + // Roots: one representative (lexicographically-smallest) name per block. + let names = block_display_names(&env, env_path); + let mut roots: Vec<(String, u32)> = names + .iter() + .filter_map(|(a, s)| id_of.get(a).map(|&id| (s.clone(), id))) + .collect(); + roots.sort(); + eprintln!( + "[sweep] {} blocks, {} roots ({:.1?} setup)", + n, + roots.len(), + t0.elapsed() + ); + + // Hot blocks to track membership for: top `top_m` by marginal prove cost. + let top_m = top_m.min(64); + let mut hot: Vec = (0..n).collect(); + hot.sort_by(|&a, &b| { + aiur_block_prove_secs(&blocks[b]) + .total_cmp(&aiur_block_prove_secs(&blocks[a])) + }); + hot.truncate(top_m); + let mut hot_bit = vec![u64::MAX; n]; // MAX = not tracked + for (j, &b) in hot.iter().enumerate() { + hot_bit[b] = j as u64; + } + + // Per-root BFS with an epoch-marked visited array per rayon worker. + let sweep_start = Instant::now(); + let rows: Vec = roots + .par_iter() + .map_init( + || (vec![0u32; n], 0u32, Vec::::new()), + |(visited, epoch, stack), (name, root)| { + *epoch += 1; + stack.clear(); + stack.push(*root); + visited[*root as usize] = *epoch; + let (mut cb, mut bytes, mut hb, mut subst, mut uniq) = + (0u32, 0u64, 0u64, 0u64, 0u64); + let (mut whnf, mut def_eq, mut nat) = (0u64, 0u64, 0u64); + let mut hot_mask = 0u64; + while let Some(b) = stack.pop() { + let e = &blocks[b as usize]; + cb += 1; + bytes += u64::from(e.serialized_size); + hb += e.heartbeats; + subst += e.subst; + uniq += e.subst_unique; + whnf += e.whnf; + def_eq += e.def_eq; + nat += e.nat_arith; + if hot_bit[b as usize] != u64::MAX { + hot_mask |= 1 << hot_bit[b as usize]; + } + for &r in profile.refs(b) { + if visited[r as usize] != *epoch { + visited[r as usize] = *epoch; + stack.push(r); + } + } + } + SweepRow { + name: name.clone(), + closure_blocks: cb, + bytes, + hb, + subst, + subst_unique: uniq, + whnf, + def_eq, + nat_arith: nat, + exec_s: aiur_exec_secs(def_eq), + exec_gib: aiur_exec_ram_gib(bytes, def_eq), + prove_s: aiur_prove_secs(bytes, subst, def_eq), + prove_gib: aiur_ram_gib(bytes, subst, def_eq), + hot_mask, + } + }, + ) + .collect(); + eprintln!( + "[sweep] {} closures walked in {:.1?}", + rows.len(), + sweep_start.elapsed() + ); + + // CSV (quote names defensively; Lean names can contain most anything). + let mut csv = String::with_capacity(rows.len() * 96); + csv.push_str( + "name,closure_blocks,bytes,hb,subst,subst_unique,whnf,def_eq,nat_arith,\ + exec_s,exec_ram_gib,prove_s,prove_ram_gib\n", + ); + for r in &rows { + let quoted = if r.name.contains(',') || r.name.contains('"') { + format!("\"{}\"", r.name.replace('"', "\"\"")) + } else { + r.name.clone() + }; + csv.push_str(&format!( + "{},{},{},{},{},{},{},{},{},{:.3},{:.2},{:.3},{:.2}\n", + quoted, + r.closure_blocks, + r.bytes, + r.hb, + r.subst, + r.subst_unique, + r.whnf, + r.def_eq, + r.nat_arith, + r.exec_s, + r.exec_gib, + r.prove_s, + r.prove_gib, + )); + } + std::fs::write(csv_out, &csv).map_err(|e| format!("write {csv_out}: {e}"))?; + + // ── Report 1: feasibility at the budget ── + let cap = budget_gib * AIUR_RAM_USABLE_FRAC; + let exec_ok = rows.iter().filter(|r| r.exec_gib <= cap).count(); + let prove_ok = rows.iter().filter(|r| r.prove_gib <= cap).count(); + let total = rows.len(); + let usable_pct = AIUR_RAM_USABLE_FRAC * 100.0; + let mut report = format!( + "sweep: {total} roots → {csv_out}\n\ + feasibility at {budget_gib:.0} GiB (cap {cap:.1} at {usable_pct:.0}% usable): \ + executable {exec_ok}/{total}, provable {prove_ok}/{total}\n\ + cheapest prove-infeasible closures (minimal bottleneck reproducers):\n", + ); + let mut infeasible: Vec<&SweepRow> = + rows.iter().filter(|r| r.prove_gib > cap).collect(); + infeasible.sort_by(|a, b| a.prove_gib.total_cmp(&b.prove_gib)); + for r in infeasible.iter().take(15) { + report.push_str(&format!( + " {:>7.1} GiB {:>8.1} s [{} blocks] {}\n", + r.prove_gib, r.prove_s, r.closure_blocks, r.name + )); + } + + // ── Report 2: cheapest closure containing each hot block ── + let names_by_id: Vec<&str> = (0..n) + .map(|i| names.get(&blocks[i].addr).map_or("", String::as_str)) + .collect(); + report.push_str( + "min-root per expensive block (cheapest closure containing it):\n", + ); + for (j, &b) in hot.iter().enumerate() { + let best = rows + .iter() + .filter(|r| r.hot_mask & (1 << j) != 0) + .min_by(|a, x| a.prove_s.total_cmp(&x.prove_s)); + match best { + Some(r) => report.push_str(&format!( + " {:>8.1} s marginal {:>8.1} s closure {} ⇐ {}\n", + aiur_block_prove_secs(&blocks[b]), + r.prove_s, + names_by_id[b], + r.name, + )), + None => report.push_str(&format!( + " {:>8.1} s marginal {} ⇐ (no named closure reaches it)\n", + aiur_block_prove_secs(&blocks[b]), + names_by_id[b], + )), + } + } + + // ── Report 3: diverse feature-mix representatives (prove-feasible) ── + if reps_k > 0 { + let cands: Vec<&SweepRow> = rows + .iter() + .filter(|r| r.prove_gib <= cap && r.closure_blocks > 1) + .collect(); + if !cands.is_empty() { + let mix = |r: &SweepRow| -> [f64; 4] { + // Feature shares on comparable scales (weights are the models' + // nlogn slopes at a common magnitude, folded to a unit simplex). + let v = [ + r.bytes as f64, + r.hb as f64 * 20.0, + r.subst as f64, + r.nat_arith as f64 * 100.0, + ]; + let s: f64 = v.iter().sum::().max(1.0); + [v[0] / s, v[1] / s, v[2] / s, v[3] / s] + }; + let dist = |a: &[f64; 4], b: &[f64; 4]| -> f64 { + a.iter().zip(b).map(|(x, y)| (x - y) * (x - y)).sum() + }; + // Seed: highest nat-arith share (the blind-spot axis). + let mut picked: Vec = vec![ + (0..cands.len()) + .max_by(|&a, &b| mix(cands[a])[3].total_cmp(&mix(cands[b])[3])) + .unwrap_or(0), + ]; + while picked.len() < reps_k.min(cands.len()) { + let next = + (0..cands.len()).filter(|i| !picked.contains(i)).max_by(|&a, &b| { + let da = picked + .iter() + .map(|&p| dist(&mix(cands[a]), &mix(cands[p]))) + .fold(f64::INFINITY, f64::min); + let db = picked + .iter() + .map(|&p| dist(&mix(cands[b]), &mix(cands[p]))) + .fold(f64::INFINITY, f64::min); + da.total_cmp(&db) + }); + match next { + Some(i) => picked.push(i), + None => break, + } + } + report.push_str( + "diverse prove-feasible representatives (bytes/hb/subst/nat mix):\n", + ); + for &i in &picked { + let r = cands[i]; + let m = mix(r); + report.push_str(&format!( + " {:>6.1} s {:>6.1} GiB mix [{:.2} {:.2} {:.2} {:.2}] {}\n", + r.prove_s, r.prove_gib, m[0], m[1], m[2], m[3], r.name + )); + } + } + } + Ok(report) +} + +/// Which backend cost models the `ix profile` summary prints. +#[derive(Clone, Copy, PartialEq, Eq)] +enum ProfileBackend { + All, + Aiur, + Zisk, +} + +impl ProfileBackend { + fn parse(s: &str) -> Self { + match s { + "aiur" => Self::Aiur, + "zisk" => Self::Zisk, + _ => Self::All, + } + } +} + +/// Print the general-purpose cost breakdown for `ix profile` — the kernel-work +/// metrics plus the predicted per-backend cost/RAM (`backend` selects Aiur, +/// Zisk, or both) and, when `top_n > 0`, per-metric block leaderboards. +// `steps as f64` is a display-only cast for `{:.2e}` formatting; precision loss +// past 2⁵³ steps is irrelevant to a two-sig-fig estimate. +#[allow(clippy::cast_precision_loss)] +#[allow(clippy::cast_possible_truncation, clippy::cast_sign_loss)] // ms display fn print_profile_summary( env_path: &str, sink: &ProfileSink, profile: &BlockProfile, + env: &IxonEnv, + top_n: usize, + backend: ProfileBackend, ) { use ix_kernel::shard::{ COST_PER_DEF_EQ, COST_PER_INGRESS_BYTE, COST_PER_INTERN, COST_PER_SUBST, - COST_PER_WHNF, SHARD_COST_FLOOR, block_step_cost, ram_gib_for_steps, + COST_PER_WHNF, SHARD_COST_FLOOR, aiur_block_prove_secs, aiur_prove_secs, + aiur_ram_gib, block_step_cost, ram_gib_for_steps, }; - let (mut hb, mut subst, mut whnf, mut defeq, mut nat, mut intern) = - (0u64, 0u64, 0u64, 0u64, 0u64, 0u64); + let (mut hb, mut subst, mut uniq, mut whnf, mut defeq, mut nat, mut intern) = + (0u64, 0u64, 0u64, 0u64, 0u64, 0u64, 0u64); for rec in sink.records.values() { hb = hb.saturating_add(rec.fuel); subst = subst.saturating_add(rec.ops.subst_nodes); + uniq = uniq.saturating_add(rec.ops.subst_unique); whnf = whnf.saturating_add(rec.ops.whnf_calls); defeq = defeq.saturating_add(rec.ops.def_eq_calls); nat = nat.saturating_add(rec.ops.nat_arith); @@ -2261,35 +2721,85 @@ fn print_profile_summary( } let ingress: u64 = profile.blocks().iter().map(|b| u64::from(b.serialized_size)).sum(); - let steps: u64 = profile - .blocks() - .iter() - .map(block_step_cost) - .fold(SHARD_COST_FLOOR, u64::saturating_add); - let ram_gib = ram_gib_for_steps(steps); - let warn = if ram_gib > 250.0 { - " → exceeds a 250 GiB box; shard it (ix shard --max-ram G)" - } else { - "" - }; eprintln!( "\n── ix profile: {env_path} ──\n\ constants {} blocks {}\n\n\ kernel work\n\ \u{20}\u{20}heartbeats {hb:>14}\n\ \u{20}\u{20}subst nodes {subst:>14}\n\ + \u{20}\u{20}uniq subst {uniq:>14}\n\ \u{20}\u{20}whnf calls {whnf:>14}\n\ \u{20}\u{20}def-eq calls {defeq:>14}\n\ \u{20}\u{20}nat-arith {nat:>14}\n\ \u{20}\u{20}intern nodes {intern:>14}\n\ - \u{20}\u{20}ingress bytes {ingress:>14}\n\n\ - predicted Zisk leaf ({SHARD_COST_FLOOR} + {COST_PER_SUBST}·subst + {COST_PER_WHNF}·whnf + {COST_PER_DEF_EQ}·def_eq + {COST_PER_INTERN}·intern; cross-shard + {COST_PER_INGRESS_BYTE}·bytes)\n\ - \u{20}\u{20}cost units ≈ {:.2e} (~92.5/guest step)\n\ - \u{20}\u{20}RAM ≈ {ram_gib:.0} GiB{warn}", + \u{20}\u{20}ingress bytes {ingress:>14}", sink.records.len(), profile.num_blocks(), - steps as f64, ); + if backend != ProfileBackend::Zisk { + // Whole-env single-run prediction; large envs exceed any real box and + // need `ix shard --backend aiur --max-ram G`. + let prove_s = aiur_prove_secs(ingress, subst, defeq); + let ram = aiur_ram_gib(ingress, subst, defeq); + let warn = if ram > 250.0 { + " → exceeds a 250 GiB box; shard it (ix shard --backend aiur --max-ram G)" + } else { + "" + }; + eprintln!( + "\n predicted Aiur single run (calibrated on the aiur bench suite; \ + limb-arithmetic-heavy work under-predicts)\n\ + \u{20}\u{20}prove ≈ {prove_s:.1} s\n\ + \u{20}\u{20}RAM ≈ {ram:.0} GiB{warn}" + ); + } + if backend != ProfileBackend::Aiur { + let steps: u64 = profile + .blocks() + .iter() + .map(block_step_cost) + .fold(SHARD_COST_FLOOR, u64::saturating_add); + let ram_gib = ram_gib_for_steps(steps); + let warn = if ram_gib > 250.0 { + " → exceeds a 250 GiB box; shard it (ix shard --max-ram G)" + } else { + "" + }; + eprintln!( + "\n predicted Zisk leaf ({SHARD_COST_FLOOR} + {COST_PER_SUBST}·subst + {COST_PER_WHNF}·whnf + {COST_PER_DEF_EQ}·def_eq + {COST_PER_INTERN}·intern; cross-shard + {COST_PER_INGRESS_BYTE}·bytes)\n\ + \u{20}\u{20}cost units ≈ {:.2e} (~92.5/guest step)\n\ + \u{20}\u{20}RAM ≈ {ram_gib:.0} GiB{warn}", + steps as f64, + ); + } + if top_n > 0 { + let names = block_display_names(env, env_path); + print_top_blocks("heartbeats", top_n, profile, &names, &|b| b.heartbeats); + print_top_blocks("substitution nodes", top_n, profile, &names, &|b| { + b.subst + }); + print_top_blocks("ingress bytes", top_n, profile, &names, &|b| { + u64::from(b.serialized_size) + }); + if backend != ProfileBackend::Zisk { + print_top_blocks( + "predicted Aiur prove ms (marginal)", + top_n, + profile, + &names, + &|b| (aiur_block_prove_secs(b) * 1e3) as u64, + ); + } + if backend != ProfileBackend::Aiur { + print_top_blocks( + "predicted Zisk cost units (sharding cost)", + top_n, + profile, + &names, + &block_step_cost, + ); + } + } } /// Aggregate per-constant records into a block-level [`BlockProfile`]: map each @@ -2317,6 +2827,17 @@ fn build_block_profile(env: &IxonEnv, merged: &ProfileSink) -> BlockProfile { builder.block(pblock.clone(), 0, psize, 0, OpCounts::default()); builder.delta_edge(cblock.clone(), pblock); } + // Touched sets may contain addresses outside the env (synthetic + // entries the checker consults); only env constants have a home block + // to attribute the touch to. + for t in &rec.touched { + if env.get_const(t).is_none() { + continue; + } + let (tblock, tsize) = resolve(t); + builder.block(tblock.clone(), 0, tsize, 0, OpCounts::default()); + builder.touch_edge(cblock.clone(), tblock); + } } builder.finish() } @@ -2428,6 +2949,8 @@ pub fn profile_anon_ixe( out: &str, isolate: bool, quiet: bool, + top_n: usize, + backend: &str, ) -> Result { let load_start = Instant::now(); let ixon_env = IxonEnv::get_anon_mmap(std::path::Path::new(path)) @@ -2449,8 +2972,21 @@ pub fn profile_anon_ixe( failed, run_start.elapsed() ); - let profile = build_block_profile(&env_arc, &merged); - print_profile_summary(path, &merged, &profile); + let mut profile = build_block_profile(&env_arc, &merged); + // Record the block-level reference graph so downstream planners can do + // closure accounting offline: an Aiur shard's witness ships its owned + // blocks' full reference closure, which only this graph can reproduce + // (and it is the fallback byte accounting when no touch graph exists). + let adj = build_block_ref_adjacency(&env_arc, &profile); + profile.set_ref_graph(&adj); + print_profile_summary( + path, + &merged, + &profile, + &env_arc, + top_n, + ProfileBackend::parse(backend), + ); let bytes = profile.to_bytes(); std::fs::write(out, &bytes).map_err(|e| format!("write {out}: {e}"))?; eprintln!( @@ -2476,12 +3012,18 @@ pub extern "C" fn rs_kernel_profile_anon( out_path: LeanString>, isolate: LeanBool>, quiet: LeanBool>, + top: LeanString>, + backend: LeanString>, ) -> LeanIOResult { + // Decimal string, kept ABI-simple like `rs_shard_esp`'s numeric params. + let top_n = top.to_string().parse::().unwrap_or(10); match profile_anon_ixe( &env_path.to_string(), &out_path.to_string(), isolate.to_bool(), quiet.to_bool(), + top_n, + &backend.to_string(), ) { Ok(s) => { eprintln!( @@ -2496,6 +3038,37 @@ pub extern "C" fn rs_kernel_profile_anon( } } +/// FFI: env-wide closure cost sweep (see [`profile_sweep`]). Writes the +/// per-root CSV and prints the feasibility/min-root/diversity reports to +/// stderr. Numeric params are decimal strings (ABI-simple). +#[unsafe(no_mangle)] +pub extern "C" fn rs_profile_sweep( + env_path: LeanString>, + prof_path: LeanString>, + csv_path: LeanString>, + budget_gib: LeanString>, + top_blocks: LeanString>, + reps: LeanString>, +) -> LeanIOResult { + let budget = budget_gib.to_string().parse::().unwrap_or(64.0); + let top_m = top_blocks.to_string().parse::().unwrap_or(10); + let reps_k = reps.to_string().parse::().unwrap_or(10); + match profile_sweep( + &env_path.to_string(), + &prof_path.to_string(), + &csv_path.to_string(), + budget, + top_m, + reps_k, + ) { + Ok(report) => { + eprintln!("[rs_sweep]\n{report}"); + LeanIOResult::ok(LeanOwned::box_usize(0)) + }, + Err(e) => LeanIOResult::error_string(&format!("rs_profile_sweep: {e}")), + } +} + /// FFI: partition a `.ixprof` into `num_shards` shards and write a `.ixes` /// manifest. Prints a what-if report to stderr. #[allow(clippy::cast_precision_loss)] // balance_pct is a small percentage @@ -2530,7 +3103,7 @@ pub extern "C" fn rs_shard_esp( } /// Total system RAM in GiB from `/proc/meminfo` (Linux); `None` if unreadable. -fn system_ram_gib() -> Option { +pub(crate) fn system_ram_gib() -> Option { let s = std::fs::read_to_string("/proc/meminfo").ok()?; let rest = s.lines().find_map(|l| l.strip_prefix("MemTotal:"))?; let kib: f64 = rest.trim().trim_end_matches("kB").trim().parse().ok()?; @@ -2540,7 +3113,9 @@ fn system_ram_gib() -> Option { /// FFI: partition a `.ixprof` to a per-shard cycle/RAM budget and write a /// `.ixes` manifest. `max_cycles` is a guest-STEP cap; if `ram_gb` > 0 it is /// converted via the measured prover RAM model and overrides `max_cycles`. Pass -/// "0" for both to default the budget to detected system RAM. +/// "0" for both to default the budget to detected system RAM. `backend` +/// must be "zisk" (guest-STEP cap); "aiur" is rejected — Aiur shards via +/// the measured scan on the `.ixe`, not a model packer. #[allow(clippy::cast_precision_loss)] #[unsafe(no_mangle)] pub extern "C" fn rs_shard_esp_cap( @@ -2550,6 +3125,7 @@ pub extern "C" fn rs_shard_esp_cap( balance_pct: LeanString>, parallelism: LeanString>, out_path: LeanString>, + backend: LeanString>, ) -> LeanIOResult { let mc = max_cycles.to_string().parse::().unwrap_or(0); let mut ram = ram_gb.to_string().parse::().unwrap_or(0.0); @@ -2557,6 +3133,7 @@ pub extern "C" fn rs_shard_esp_cap( parallelism.to_string().parse::().unwrap_or(1).max(1); let balance = (balance_pct.to_string().parse::().unwrap_or(5) as f64) / 100.0; + let aiur = backend.to_string() == "aiur"; // No explicit cap → default the RAM budget to detected system RAM. if mc == 0 && ram <= 0.0 { match system_ram_gib() { @@ -2573,6 +3150,11 @@ pub extern "C" fn rs_shard_esp_cap( }, } } + if aiur { + return LeanIOResult::error_string( + "the Aiur model packer was removed; run the measured scan instead: ix shard --max-ram G", + ); + } let cap = if ram > 0.0 { ix_kernel::shard::cycle_cap_for_ram(ram) } else { mc }; if cap == 0 { diff --git a/crates/ixvm-codegen/src/aiur_ixvm_runner.rs b/crates/ixvm-codegen/src/aiur_ixvm_runner.rs index 325ad48c..b749bc7b 100644 --- a/crates/ixvm-codegen/src/aiur_ixvm_runner.rs +++ b/crates/ixvm-codegen/src/aiur_ixvm_runner.rs @@ -34,10 +34,26 @@ pub fn execute_ixvm( args: Vec, io_buffer: &mut IOBuffer, ) -> Result<(QueryRecord, Vec), ExecError> { + let mut record = QueryRecord::new(toplevel); + let output = + execute_ixvm_with_record(toplevel, fun_idx, &args, io_buffer, &mut record)?; + if aiur::execute::query_stats_enabled() { + aiur::execute::dump_query_stats(&record, "ixvm final"); + } + Ok((record, output)) +} + +/// Like [`execute_ixvm`] but accumulating into a caller-owned record — +/// see `aiur::execute::Toplevel::execute_with_record`. +pub fn execute_ixvm_with_record( + toplevel: &Toplevel, + fun_idx: FunIdx, + args: &[G], + io_buffer: &mut IOBuffer, + record: &mut QueryRecord, +) -> Result, ExecError> { if !toplevel.functions[fun_idx].entry { return Err(ExecError::NotEntryFunction(fun_idx)); } - let mut record = QueryRecord::new(toplevel); - let output = execute_generated(fun_idx, &args, &mut record, io_buffer)?; - Ok((record, output)) + execute_generated(fun_idx, args, record, io_buffer) } diff --git a/crates/ixvm-codegen/src/aiur_ixvm_witness.rs b/crates/ixvm-codegen/src/aiur_ixvm_witness.rs index 062734da..7548ae55 100644 --- a/crates/ixvm-codegen/src/aiur_ixvm_witness.rs +++ b/crates/ixvm-codegen/src/aiur_ixvm_witness.rs @@ -39,12 +39,14 @@ //! (extending channel arenas + inserting into the key→(idx,len) //! map) runs serially, since the arena `idx` is monotonic. -use multi_stark::p3_field::PrimeCharacteristicRing; +use std::sync::Arc; + +use multi_stark::p3_field::{PrimeCharacteristicRing, PrimeField64}; use rayon::prelude::*; use rustc_hash::FxHashSet; use aiur::G; -use aiur::execute::{IOBuffer, IOKeyInfo}; +use aiur::execute::{IOBuffer, IOFaultSource, IOKeyInfo}; use ix_common::address::Address; use ix_common::env::ReducibilityHints; use ix_common::prim_addrs::PrimAddrs; @@ -52,6 +54,55 @@ use ixon::Env; use ixon::assumption_tree::AssumptionTree; use ixon::proof::Claim; +/// Lazy witness backing over a shared env: materializes ch 2 (constant +/// bytes), ch 3 (Defn hint), and ch 4 (blob bytes) entries on first +/// `io_read` miss instead of seeding the whole `bfs_closure` eagerly. +/// Host witness RAM then scales with the FAULTED set — what the check +/// actually touches — instead of the shipped closure, which the eager +/// path stores 8×-expanded (one `G` per byte). Soundness-neutral: the +/// kernel blake3-verifies faulted bytes against their content-addressed +/// key exactly as it does eagerly-seeded ones, and no scope widens — +/// content addressing means a key can only ever resolve to one value. +pub struct EnvFaultSource { + env: Arc, +} + +impl EnvFaultSource { + pub fn new(env: Arc) -> Arc { + Arc::new(Self { env }) + } +} + +/// Decode a 32-limb channel key back to the `Address` it spells; `None` +/// if any limb is out of byte range (such a key can name no env entry). +fn key_to_addr(key: &[G]) -> Option
{ + if key.len() != 32 { + return None; + } + let mut bytes = [0u8; 32]; + for (i, g) in key.iter().enumerate() { + let Ok(b) = u8::try_from(g.as_canonical_u64()) else { + return None; + }; + bytes[i] = b; + } + Address::from_slice(&bytes).ok() +} + +impl IOFaultSource for EnvFaultSource { + fn fault(&self, channel: G, key: &[G]) -> Option> { + let addr = key_to_addr(key)?; + match channel.as_canonical_u64() { + 2 => { + self.env.consts.get(&addr).map(|lc| bytes_to_g(lc.raw_bytes())) + }, + 3 => self.env.anon_hints.get(&addr).map(|h| vec![hint_to_g(&h)]), + 4 => self.env.blobs.get(&addr).map(|b| bytes_to_g(b.value())), + _ => None, + } + } +} + /// Append `data` to the per-channel arena and record `(idx, len)` /// in the `(channel, key)` info map. #[inline] @@ -203,10 +254,7 @@ pub fn build_claim_check_witness( let digest = Address::hash(&claim_bytes); let digest_key = addr_key(&digest); - let mut io = IOBuffer { - data: rustc_hash::FxHashMap::default(), - map: rustc_hash::FxHashMap::default(), - }; + let mut io = IOBuffer::new(); // ch 0: claim bytes extend(&mut io, G::ZERO, digest_key.clone(), bytes_to_g(&claim_bytes)); // ch 2/3/4: per-const/hint/blob entries — parallel byte conversion. @@ -279,19 +327,36 @@ pub fn build_shard_check_env_witness( env: &Env, owned: &[Address], ) -> Result<(Claim, Vec, IOBuffer), String> { - // Claim + THIN frontier from the shared convention module — the - // single source of truth for shard CheckEnv digests (see - // ixon::shard_claim; Lean mirror: - // IxVM.ClaimHarness.shardCheckEnvClaimThin). + let mut io = IOBuffer::new(); + let (claim, digest_key) = seed_shard_check_env_claim(env, owned, &mut io)?; + let byte_scope = witness_scope(env, owned); + add_entries_parallel(env, &byte_scope, &mut io); + Ok((claim, digest_key, io)) +} + +/// Seed ONLY the claim-side channels for a thin-frontier +/// `CheckEnv{owned}` claim into `io`: the claim wire (ch 0) and the +/// owned/assumption trees (ch 1). Byte channels (2/3/4) are the caller's +/// business — eagerly via `add_entries_parallel` or lazily via an +/// `EnvFaultSource` backing. Returns the claim and its digest key (the +/// `verify_claim` entry input). Claim + THIN frontier come from the +/// shared convention module — the single source of truth for shard +/// CheckEnv digests (see `ixon::shard_claim`; Lean mirror: +/// `IxVM.ClaimHarness.shardCheckEnvClaimThin`). +pub fn seed_shard_check_env_claim( + env: &Env, + owned: &[Address], + io: &mut IOBuffer, +) -> Result<(Claim, Vec), String> { let (claim, frontier) = ixon::shard_claim::shard_check_env_claim(env, owned) .ok_or_else(|| { - "build_shard_check_env_witness: empty owned set".to_string() + "seed_shard_check_env_claim: empty owned set".to_string() })?; let mut owned_sorted: Vec
= owned.to_vec(); owned_sorted.sort(); let owned_tree = AssumptionTree::canonical(&owned_sorted).ok_or_else(|| { - "build_shard_check_env_witness: empty owned set".to_string() + "seed_shard_check_env_claim: empty owned set".to_string() })?; let asm_tree = AssumptionTree::canonical(&frontier); debug_assert_eq!( @@ -305,24 +370,44 @@ pub fn build_shard_check_env_witness( claim.put(&mut claim_bytes); let digest = Address::hash(&claim_bytes); let digest_key = addr_key(&digest); - - let mut io = IOBuffer { - data: rustc_hash::FxHashMap::default(), - map: rustc_hash::FxHashMap::default(), - }; - extend(&mut io, G::ZERO, digest_key.clone(), bytes_to_g(&claim_bytes)); - let byte_scope = witness_scope(env, owned); - add_entries_parallel(env, &byte_scope, &mut io); + extend(io, G::ZERO, digest_key.clone(), bytes_to_g(&claim_bytes)); extend( - &mut io, + io, G::ONE, addr_key(&owned_tree.root()), bytes_to_g(&owned_tree.ser()), ); if let Some(at) = asm_tree { - extend(&mut io, G::ONE, addr_key(&at.root()), bytes_to_g(&at.ser())); + extend(io, G::ONE, addr_key(&at.root()), bytes_to_g(&at.ser())); } + Ok((claim, digest_key)) +} +/// Lazy-witness variant of [`build_shard_check_env_witness`]: claim +/// channels seeded eagerly, byte channels served on demand by an +/// [`EnvFaultSource`] over the shared env. Witness RAM ∝ faulted set. +pub fn build_shard_check_env_witness_lazy( + env: &Arc, + owned: &[Address], +) -> Result<(Claim, Vec, IOBuffer), String> { + let mut io = IOBuffer::with_backing(EnvFaultSource::new(env.clone())); + let (claim, digest_key) = seed_shard_check_env_claim(env, owned, &mut io)?; + Ok((claim, digest_key, io)) +} + +/// Lazy-witness variant of [`build_claim_check_witness`]: only the claim +/// wire is seeded; constant/hint/blob bytes fault in on demand. +pub fn build_claim_check_witness_lazy( + env: &Arc, + target: &Address, +) -> Result<(Claim, Vec, IOBuffer), String> { + let claim = Claim::Check { const_addr: target.clone(), assumptions: None }; + let mut claim_bytes: Vec = Vec::new(); + claim.put(&mut claim_bytes); + let digest = Address::hash(&claim_bytes); + let digest_key = addr_key(&digest); + let mut io = IOBuffer::with_backing(EnvFaultSource::new(env.clone())); + extend(&mut io, G::ZERO, digest_key.clone(), bytes_to_g(&claim_bytes)); Ok((claim, digest_key, io)) } diff --git a/crates/ixvm-codegen/src/aiur_multi_stark_runner.rs b/crates/ixvm-codegen/src/aiur_multi_stark_runner.rs index 720fce56..5365e196 100644 --- a/crates/ixvm-codegen/src/aiur_multi_stark_runner.rs +++ b/crates/ixvm-codegen/src/aiur_multi_stark_runner.rs @@ -17,7 +17,6 @@ //! buffer across FFI. use multi_stark::p3_field::PrimeCharacteristicRing; -use rustc_hash::FxHashMap; use crate::aiur_multi_stark::execute_generated; use aiur::G; @@ -60,7 +59,7 @@ pub fn verifier_io_buffer(proof: &[u8], vk: &[u8], claims: &[u8]) -> IOBuffer { let _ = std::fs::write(format!("{dir}/claims.bin"), claims); } let mut io = - IOBuffer { data: FxHashMap::default(), map: FxHashMap::default() }; + IOBuffer::new(); for (channel, bytes) in [(0u64, proof), (1, vk), (2, claims)].map(|(c, b)| (G::from_u64(c), b)) { diff --git a/crates/ixvm-codegen/src/env_handle.rs b/crates/ixvm-codegen/src/env_handle.rs index 8a5168dc..fb0dd1e6 100644 --- a/crates/ixvm-codegen/src/env_handle.rs +++ b/crates/ixvm-codegen/src/env_handle.rs @@ -11,17 +11,22 @@ //! `Env::anon_hints` map — both readers used here populate the map //! directly, so no post-decode harvest is needed. +use std::sync::Arc; + use ixon::Env; +/// The env is held behind an `Arc` so lazy witness backings +/// (`aiur_ixvm_witness::EnvFaultSource`) can hold a clone that outlives +/// any particular borrow of the handle. pub struct EnvHandle { - pub env: Env, + pub env: Arc, } impl EnvHandle { /// Load via `Env::get_anon_mmap` (zero-copy mmap of the `.ixe` file). pub fn from_ixe_path(path: &std::path::Path) -> Result { let env = Env::get_anon_mmap(path)?; - Ok(Self { env }) + Ok(Self { env: Arc::new(env) }) } /// Decode a serialized env blob (`Ixon.serEnv` output) via @@ -30,6 +35,6 @@ impl EnvHandle { pub fn from_bytes(bytes: &[u8]) -> Result { let mut cursor: &[u8] = bytes; let env = Env::get(&mut cursor)?; - Ok(Self { env }) + Ok(Self { env: Arc::new(env) }) } } diff --git a/crates/kernel/src/profile.rs b/crates/kernel/src/profile.rs index 1fb71613..7d8b881e 100644 --- a/crates/kernel/src/profile.rs +++ b/crates/kernel/src/profile.rs @@ -53,6 +53,10 @@ use ix_common::address::Address; #[cfg(not(target_os = "zkvm"))] thread_local! { static SUBST_NODES: Cell = const { Cell::new(0) }; + static SUBST_UNIQUE: Cell = const { Cell::new(0) }; + static SUBST_CTX: Cell = const { Cell::new(0) }; + static SUBST_SEEN: std::cell::RefCell> = + std::cell::RefCell::new(FxHashSet::default()); static WHNF_CALLS: Cell = const { Cell::new(0) }; static DEF_EQ_CALLS: Cell = const { Cell::new(0) }; static NAT_ARITH: Cell = const { Cell::new(0) }; @@ -66,6 +70,48 @@ pub fn bump_subst_nodes() { SUBST_NODES.with(|c| c.set(c.get().wrapping_add(1))); } +/// Set the substitution-context component of the unique-work key: a fold +/// of the substitution arguments' identities, computed once per top-level +/// `instantiate_rev` call (the recursion never re-enters subst, so one +/// slot suffices). Mixed into every node key by [`bump_subst_unique`]. +#[inline(always)] +pub fn set_subst_ctx(ctx: u64) { + #[cfg(not(target_os = "zkvm"))] + SUBST_CTX.with(|c| c.set(ctx)); + #[cfg(target_os = "zkvm")] + let _ = ctx; +} + +/// Count one substitution-node visit deduplicated by its work identity: +/// (expression `expr_key`, binder `depth`, the current substitution +/// context from [`set_subst_ctx`]). A memoizing executor (Aiur proves +/// each unique query once; repeats are memo-table lookups) pays only for +/// distinct work, so `subst_unique`, not the raw visit count, is the +/// substitution-volume feature for an Aiur cost model. The seen-set +/// spans one constant's check — the same scope as the other counters +/// (cleared by [`take_op_counts`]). +#[inline(always)] +pub fn bump_subst_unique(expr_key: u64, depth: u64) { + #[cfg(not(target_os = "zkvm"))] + { + // splitmix64 finalizer over the mixed triple — collisions only cost + // model accuracy, never soundness. + let mut k = expr_key + ^ SUBST_CTX.with(Cell::get) + ^ depth.wrapping_mul(0x9E37_79B9_7F4A_7C15); + k = (k ^ (k >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + k = (k ^ (k >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + k ^= k >> 31; + SUBST_SEEN.with(|s| { + if s.borrow_mut().insert(k) { + SUBST_UNIQUE.with(|c| c.set(c.get().wrapping_add(1))); + } + }); + } + #[cfg(target_os = "zkvm")] + let _ = (expr_key, depth); +} + /// Count one `whnf` entry. #[inline(always)] pub fn bump_whnf() { @@ -106,6 +152,9 @@ pub fn bump_intern_nodes() { #[derive(Default, Debug, Clone, Copy)] pub struct OpCounts { pub subst_nodes: u64, + /// Distinct substitution work items (see [`bump_subst_unique`]) — the + /// post-memoization substitution volume a memoizing executor pays. + pub subst_unique: u64, pub whnf_calls: u64, pub def_eq_calls: u64, pub nat_arith: u64, @@ -116,6 +165,7 @@ impl OpCounts { /// Saturating field-wise accumulation. pub fn add(&mut self, o: &OpCounts) { self.subst_nodes = self.subst_nodes.saturating_add(o.subst_nodes); + self.subst_unique = self.subst_unique.saturating_add(o.subst_unique); self.whnf_calls = self.whnf_calls.saturating_add(o.whnf_calls); self.def_eq_calls = self.def_eq_calls.saturating_add(o.def_eq_calls); self.nat_arith = self.nat_arith.saturating_add(o.nat_arith); @@ -127,8 +177,10 @@ impl OpCounts { pub fn take_op_counts() -> OpCounts { #[cfg(not(target_os = "zkvm"))] { + SUBST_SEEN.with(|s| s.borrow_mut().clear()); OpCounts { subst_nodes: SUBST_NODES.with(|c| c.replace(0)), + subst_unique: SUBST_UNIQUE.with(|c| c.replace(0)), whnf_calls: WHNF_CALLS.with(|c| c.replace(0)), def_eq_calls: DEF_EQ_CALLS.with(|c| c.replace(0)), nat_arith: NAT_ARITH.with(|c| c.replace(0)), @@ -142,7 +194,7 @@ pub fn take_op_counts() -> OpCounts { /// Magic bytes at the head of every `.ixprof` file. const MAGIC: &[u8; 8] = b"IXPROF\0\0"; /// On-disk format version. Bump on any incompatible layout change. -const VERSION: u32 = 2; +const VERSION: u32 = 3; /// Per-block recorded statistics. #[derive(Clone, Debug, PartialEq, Eq)] @@ -159,6 +211,10 @@ pub struct BlockEntry { /// dominant reduction-volume cost driver. Recorded when profiled with op /// counters enabled; 0 otherwise. pub subst: u64, + /// Distinct substitution work items checking this block — the + /// post-memoization substitution volume (see `bump_subst_unique`); + /// the Aiur-relevant counterpart of `subst`. + pub subst_unique: u64, /// `whnf` entries checking this block (counted before cache probes). pub whnf: u64, /// `is_def_eq` entries checking this block. @@ -182,6 +238,27 @@ pub struct BlockProfile { delta_row: Vec, /// CSR column indices: producer block ids, grouped by consumer. delta_col: Vec, + /// CSR row offsets into `ref_col`, length `blocks.len() + 1`. The + /// **reference** graph (every cross-block `Constant.refs` edge, projections + /// and mutual members folded into home blocks) — a superset of the delta + /// graph. Reachability over it is a block's full dependency closure, which + /// is what an Aiur shard witness ships (`witness_scope` is the owned + /// blocks' whole closure), so the Aiur packer's byte accounting runs on + /// this graph, not the delta graph. + ref_row: Vec, + /// CSR column indices: referenced block ids, grouped by referrer. + ref_col: Vec, + /// CSR row offsets into `touch_col`, length `blocks.len() + 1`. + /// + /// The **touch** graph: blocks whose constants the kernel CONSULTED + /// (`try_get_const`) while checking each block's members — whether a body + /// was unfolded or only a type read. Unlike the delta and reference + /// graphs this is a *measurement* of the ingress a lazy checker demands, + /// not a prediction from graph structure: it is the set the check + /// actually faulted in. Self-edges dropped; recorded per profiling run. + touch_row: Vec, + /// CSR column indices: touched block ids, grouped by consumer. + touch_col: Vec, } impl BlockProfile { @@ -243,11 +320,59 @@ impl BlockProfile { self.blocks.iter().map(|b| u128::from(b.heartbeats)).sum() } + /// Whether the reference graph is present (older recordings may lack it). + pub fn has_ref_graph(&self) -> bool { + !self.ref_row.is_empty() + } + + /// Referenced block ids of block `b` (sorted, deduped, no self-edges). + /// Empty when no reference graph was recorded. + pub fn refs(&self, b: u32) -> &[u32] { + if self.ref_row.is_empty() { + return &[]; + } + let lo = self.ref_row[b as usize]; + let hi = self.ref_row[b as usize + 1]; + &self.ref_col[lo..hi] + } + + /// Whether the touch graph is present (profiles recorded before touch + /// recording, or built without a sink, lack it). + pub fn has_touch_graph(&self) -> bool { + !self.touch_row.is_empty() + } + + /// Block ids consulted while checking block `b` (sorted, deduped, no + /// self-edges). Empty when no touch graph was recorded. This is the + /// measured ingress set of `b` minus `b` itself: what a lazy checker + /// faulted in to check it. + pub fn touched_blocks(&self, b: u32) -> &[u32] { + if self.touch_row.is_empty() { + return &[]; + } + let lo = self.touch_row[b as usize]; + let hi = self.touch_row[b as usize + 1]; + &self.touch_col[lo..hi] + } + + /// Attach the block-level reference graph (per-block sorted, deduped, + /// self-edge-free referenced ids; one row per block). + pub fn set_ref_graph(&mut self, adj: &[Vec]) { + assert_eq!(adj.len(), self.blocks.len()); + self.ref_row = Vec::with_capacity(adj.len() + 1); + self.ref_row.push(0); + self.ref_col = Vec::with_capacity(adj.iter().map(Vec::len).sum()); + for row in adj { + self.ref_col.extend_from_slice(row); + self.ref_row.push(self.ref_col.len()); + } + } + /// Serialize to the `.ixprof` binary format. pub fn to_bytes(&self) -> Vec { let n = self.blocks.len(); let mut out = Vec::with_capacity( - 8 + 4 + 4 + n * 80 + 8 + (n + 1) * 8 + self.delta_col.len() * 4, + 8 + 4 + 4 + n * 96 + 8 + (n + 1) * 8 + self.delta_col.len() * 4, ); out.extend_from_slice(MAGIC); out.extend_from_slice(&VERSION.to_le_bytes()); @@ -258,18 +383,25 @@ impl BlockProfile { out.extend_from_slice(&b.serialized_size.to_le_bytes()); out.extend_from_slice(&b.const_count.to_le_bytes()); out.extend_from_slice(&b.subst.to_le_bytes()); + out.extend_from_slice(&b.subst_unique.to_le_bytes()); out.extend_from_slice(&b.whnf.to_le_bytes()); out.extend_from_slice(&b.def_eq.to_le_bytes()); out.extend_from_slice(&b.nat_arith.to_le_bytes()); out.extend_from_slice(&b.intern.to_le_bytes()); } - out.extend_from_slice(&(self.delta_col.len() as u64).to_le_bytes()); - // CSR row offsets (n+1 entries) as u64. - for &off in &self.delta_row { - out.extend_from_slice(&(off as u64).to_le_bytes()); - } - for &p in &self.delta_col { - out.extend_from_slice(&p.to_le_bytes()); + // CSR sections. The delta graph is always present; the reference and + // touch graphs are each framed by a one-byte presence flag so either + // can be absent without EOF inference. + write_csr(&mut out, &self.delta_row, &self.delta_col); + for (row, col) in + [(&self.ref_row, &self.ref_col), (&self.touch_row, &self.touch_col)] + { + if row.is_empty() { + out.push(0); + } else { + out.push(1); + write_csr(&mut out, row, col); + } } out } @@ -294,6 +426,7 @@ impl BlockProfile { let serialized_size = r.u32()?; let const_count = r.u32()?; let subst = r.u64()?; + let subst_unique = r.u64()?; let whnf = r.u64()?; let def_eq = r.u64()?; let nat_arith = r.u64()?; @@ -304,39 +437,32 @@ impl BlockProfile { serialized_size, const_count, subst, + subst_unique, whnf, def_eq, nat_arith, intern, }); } - let num_edges = r.u64()? as usize; - let mut delta_row = Vec::with_capacity(n + 1); - for _ in 0..n + 1 { - delta_row.push(r.u64()? as usize); - } - let mut delta_col = Vec::with_capacity(num_edges); - for _ in 0..num_edges { - delta_col.push(r.u32()?); - } - // Structural validation: monotone offsets bounded by edge count, in-range ids. - if delta_row.len() != n + 1 - || delta_row.first() != Some(&0) - || delta_row.last() != Some(&num_edges) - { - return Err(ProfileError::Corrupt); - } - for w in delta_row.windows(2) { - if w[0] > w[1] { - return Err(ProfileError::Corrupt); + let (delta_row, delta_col) = read_csr(&mut r, n)?; + let mut optional = [(Vec::new(), Vec::new()), (Vec::new(), Vec::new())]; + for slot in &mut optional { + match r.u8()? { + 0 => {}, + 1 => *slot = read_csr(&mut r, n)?, + _ => return Err(ProfileError::Corrupt), } } - for &p in &delta_col { - if p as usize >= n { - return Err(ProfileError::Corrupt); - } - } - Ok(BlockProfile { blocks, delta_row, delta_col }) + let [(ref_row, ref_col), (touch_row, touch_col)] = optional; + Ok(BlockProfile { + blocks, + delta_row, + delta_col, + ref_row, + ref_col, + touch_row, + touch_col, + }) } } @@ -383,6 +509,9 @@ impl<'a> Reader<'a> { self.pos = end; Ok(s) } + fn u8(&mut self) -> Result { + Ok(self.take(1)?[0]) + } fn u32(&mut self) -> Result { Ok(u32::from_le_bytes(self.take(4)?.try_into().unwrap())) } @@ -391,6 +520,42 @@ impl<'a> Reader<'a> { } } +/// Serialize one CSR section: edge count, `n+1` u64 row offsets, u32 columns. +fn write_csr(out: &mut Vec, row: &[usize], col: &[u32]) { + out.extend_from_slice(&(col.len() as u64).to_le_bytes()); + for &off in row { + out.extend_from_slice(&(off as u64).to_le_bytes()); + } + for &c in col { + out.extend_from_slice(&c.to_le_bytes()); + } +} + +/// Read and structurally validate one CSR section over `n` blocks: offsets +/// monotone from 0 to the edge count, column ids in range. +fn read_csr( + r: &mut Reader<'_>, + n: usize, +) -> Result<(Vec, Vec), ProfileError> { + let num_edges = r.u64()? as usize; + let mut row = Vec::with_capacity(n + 1); + for _ in 0..n + 1 { + row.push(r.u64()? as usize); + } + let mut col = Vec::with_capacity(num_edges); + for _ in 0..num_edges { + col.push(r.u32()?); + } + if row.first() != Some(&0) + || row.last() != Some(&num_edges) + || row.windows(2).any(|w| w[0] > w[1]) + || col.iter().any(|&x| x as usize >= n) + { + return Err(ProfileError::Corrupt); + } + Ok((row, col)) +} + /// Accumulates block-level statistics and delta edges (keyed by address), then /// freezes into a [`BlockProfile`] with stable, address-sorted block ids. /// @@ -411,6 +576,7 @@ struct Accum { const_count: u32, ops: OpCounts, producers: FxHashSet
, + touched: FxHashSet
, } impl ProfileBuilder { @@ -448,6 +614,17 @@ impl ProfileBuilder { self.blocks.entry(consumer).or_default().producers.insert(producer); } + /// Record that checking `consumer` consulted `target` (`try_get_const`), + /// whether its body was unfolded or only its type read. Self-edges are + /// ignored; both endpoints are ensured as blocks, like [`Self::delta_edge`]. + pub fn touch_edge(&mut self, consumer: Address, target: Address) { + if consumer == target { + return; + } + self.blocks.entry(target.clone()).or_default(); + self.blocks.entry(consumer).or_default().touched.insert(target); + } + /// Freeze into an immutable [`BlockProfile`]. Block ids are assigned by /// sorting addresses, so the result is deterministic regardless of insertion /// order. @@ -461,6 +638,10 @@ impl ProfileBuilder { let mut delta_row = Vec::with_capacity(addrs.len() + 1); let mut delta_col = Vec::new(); delta_row.push(0usize); + let mut touch_row = Vec::with_capacity(addrs.len() + 1); + let mut touch_col = Vec::new(); + touch_row.push(0usize); + let any_touches = self.blocks.values().any(|a| !a.touched.is_empty()); for addr in &addrs { let a = &self.blocks[addr]; @@ -470,6 +651,7 @@ impl ProfileBuilder { serialized_size: a.serialized_size, const_count: a.const_count, subst: a.ops.subst_nodes, + subst_unique: a.ops.subst_unique, whnf: a.ops.whnf_calls, def_eq: a.ops.def_eq_calls, nat_arith: a.ops.nat_arith, @@ -480,9 +662,28 @@ impl ProfileBuilder { prods.dedup(); delta_col.extend_from_slice(&prods); delta_row.push(delta_col.len()); + let mut touches: Vec = a.touched.iter().map(|t| id_of[t]).collect(); + touches.sort_unstable(); + touches.dedup(); + touch_col.extend_from_slice(&touches); + touch_row.push(touch_col.len()); + } + // A profile recorded without touch instrumentation has no touch graph + // at all, which readers distinguish from "recorded, all rows empty". + if !any_touches { + touch_row = Vec::new(); + touch_col = Vec::new(); } - BlockProfile { blocks, delta_row, delta_col } + BlockProfile { + blocks, + delta_row, + delta_col, + ref_row: Vec::new(), + ref_col: Vec::new(), + touch_row, + touch_col, + } } } @@ -508,6 +709,10 @@ pub struct ConstRecord { pub fuel: u64, /// Constant addresses whose bodies were delta-unfolded during the check. pub producers: FxHashSet
, + /// Constant addresses CONSULTED during the check (`try_get_const`) — + /// a superset of `producers` that also covers type-only reads. The + /// measured ingress set of a lazy checker. + pub touched: FxHashSet
, /// Richer cost features (substitution-node visits, whnf/def-eq calls), /// recorded on every native profiling run; compiled out (all zero) on the /// zkvm target. @@ -520,17 +725,20 @@ impl ProfileSink { } /// Accumulate one constant's record (additive in fuel + op counts, set-union - /// in producers) so repeated flushes for the same constant combine correctly. + /// in producers/touched) so repeated flushes for the same constant combine + /// correctly. pub fn record( &mut self, consumer: Address, fuel: u64, producers: impl IntoIterator, + touched: impl IntoIterator, ops: OpCounts, ) { let rec = self.records.entry(consumer).or_default(); rec.fuel = rec.fuel.saturating_add(fuel); rec.producers.extend(producers); + rec.touched.extend(touched); rec.ops.add(&ops); } @@ -540,6 +748,7 @@ impl ProfileSink { let e = self.records.entry(addr).or_default(); e.fuel = e.fuel.saturating_add(rec.fuel); e.producers.extend(rec.producers); + e.touched.extend(rec.touched); e.ops.add(&rec.ops); } } @@ -568,6 +777,11 @@ mod tests { b.delta_edge(addr(1), addr(3)); b.delta_edge(addr(3), addr(2)); b.delta_edge(addr(2), addr(2)); + // a consulted c; b consulted a and c; self-edge ignored. + b.touch_edge(addr(1), addr(3)); + b.touch_edge(addr(2), addr(1)); + b.touch_edge(addr(2), addr(3)); + b.touch_edge(addr(3), addr(3)); b.finish() } @@ -606,12 +820,36 @@ mod tests { assert_eq!(got, vec![0, 2]); } + #[test] + fn touch_graph_sorted_self_edge_dropped_and_absent_when_unrecorded() { + let p = sample(); + assert!(p.has_touch_graph()); + // block 0 (addr 1) consulted block 2; block 1 consulted blocks 0 and 2. + assert_eq!(p.touched_blocks(0), &[2]); + assert_eq!(p.touched_blocks(1), &[0, 2]); + // block 2 (addr 3): self-edge dropped → no touches. + assert_eq!(p.touched_blocks(2), &[]); + // A builder fed no touch edges yields no touch graph at all. + let mut b = ProfileBuilder::new(); + b.block(addr(1), 1, 1, 1, ops(0)); + let q = b.finish(); + assert!(!q.has_touch_graph()); + assert_eq!(q.touched_blocks(0), &[]); + } + #[test] fn roundtrip_serialization() { let p = sample(); let bytes = p.to_bytes(); let q = BlockProfile::from_bytes(&bytes).unwrap(); assert_eq!(p, q); + // Absent optional sections roundtrip as absent, not as empty-present. + let mut b = ProfileBuilder::new(); + b.block(addr(1), 1, 1, 1, ops(0)); + let bare = b.finish(); + let r = BlockProfile::from_bytes(&bare.to_bytes()).unwrap(); + assert_eq!(bare, r); + assert!(!r.has_touch_graph() && !r.has_ref_graph()); } #[test] @@ -634,12 +872,16 @@ mod tests { // via separate builders merged conceptually; result must be identical. let mut b = ProfileBuilder::new(); b.delta_edge(addr(3), addr(2)); + b.touch_edge(addr(2), addr(3)); b.block(addr(3), 300, 30, 1, ops(150)); b.delta_edge(addr(1), addr(3)); + b.touch_edge(addr(3), addr(3)); b.block(addr(2), 200, 20, 3, ops(100)); b.delta_edge(addr(1), addr(2)); + b.touch_edge(addr(2), addr(1)); b.block(addr(1), 100, 10, 1, ops(50)); b.delta_edge(addr(2), addr(2)); + b.touch_edge(addr(1), addr(3)); assert_eq!(b.finish(), sample()); } } diff --git a/crates/kernel/src/shard.rs b/crates/kernel/src/shard.rs index abdf4cc9..b8711295 100644 --- a/crates/kernel/src/shard.rs +++ b/crates/kernel/src/shard.rs @@ -1258,6 +1258,79 @@ fn uncoarsen_refine( // Manifest // ============================================================================ +/// Planner cost of one shard, tagged by the metric it is denominated in. +/// All shards of one manifest carry the same variant (one packer, one +/// backend); the tag makes the unit explicit — cross-backend values can +/// never be compared by accident — and puts the heaviest-first prove +/// ordering in the manifest itself instead of a sidecar file. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Default)] +pub enum ShardCost { + /// No planner cost recorded. + #[default] + Unknown, + /// Sum of member kernel heartbeats — the balance metric of the + /// profile-driven min-cut partitioners. + ProfileHeartbeats(u64), + /// Zisk guest cost units (ziskemu `-X` TOTAL; see the calibrated cost + /// model below). + ZiskCostUnits(u64), + /// Aiur circuit FFT cost (raw `w·h·log2(h)` units, the same + /// denomination as the bench rows' `fft-cost` and the Lean stats + /// dump): MEASURED when produced by the env scan, model-predicted + /// when produced by the `.ixprof` packer. + AiurFft(u64), +} + +/// An FFT cost (raw `w·h·log2(h)` units, f64) as a saturating `u64`, +/// converted without a lossy `as` cast (decimal round-trip; cost +/// magnitudes sit far below any precision edge, and this only runs once +/// per shard at manifest-write time). +pub fn cost_fft(fft: f64) -> u64 { + format!("{:.0}", fft.max(0.0)).parse().unwrap_or(u64::MAX) +} + +impl ShardCost { + /// The scalar used for ordering and balance WITHIN one manifest; its + /// unit is whatever the variant says. + pub fn value(self) -> u64 { + match self { + Self::Unknown => 0, + Self::ProfileHeartbeats(v) + | Self::ZiskCostUnits(v) + | Self::AiurFft(v) => v, + } + } + + /// Unit label for reports. + pub fn unit(self) -> &'static str { + match self { + Self::Unknown => "", + Self::ProfileHeartbeats(_) => "hb", + Self::ZiskCostUnits(_) => "cost-units", + Self::AiurFft(_) => "fft", + } + } + + fn tag(self) -> u8 { + match self { + Self::Unknown => 0, + Self::ProfileHeartbeats(_) => 1, + Self::ZiskCostUnits(_) => 2, + Self::AiurFft(_) => 3, + } + } + + fn from_tag(tag: u8, value: u64) -> Result { + match tag { + 0 => Ok(Self::Unknown), + 1 => Ok(Self::ProfileHeartbeats(value)), + 2 => Ok(Self::ZiskCostUnits(value)), + 3 => Ok(Self::AiurFft(value)), + t => Err(format!("unknown shard-cost tag {t}")), + } + } +} + /// Per-shard summary in a [`ShardManifest`]. #[derive(Clone, Debug, PartialEq, Eq)] pub struct ShardInfo { @@ -1265,8 +1338,8 @@ pub struct ShardInfo { pub id: u32, /// Member block addresses. pub blocks: Vec
, - /// Sum of member heartbeats (balance metric). - pub heartbeats: u64, + /// Planner cost (packing/balance metric, unit-tagged). + pub cost: ShardCost, /// Sum of member serialized sizes (the shard's own ingress). pub own_size: u64, /// Foreign blocks delta-unfolded by members but proven in other shards. @@ -1326,6 +1399,7 @@ impl ShardManifest { members[s].iter().map(|&b| profile.block(b).addr.clone()).collect(); let heartbeats: u64 = members[s].iter().map(|&b| profile.block(b).heartbeats).sum(); + let cost = ShardCost::ProfileHeartbeats(heartbeats); let own_size: u64 = members[s] .iter() .map(|&b| u64::from(profile.block(b).serialized_size)) @@ -1340,7 +1414,7 @@ impl ShardManifest { shards.push(ShardInfo { id: s as u32, blocks, - heartbeats, + cost, own_size, foreign_blocks, cross_ingress, @@ -1365,16 +1439,22 @@ impl ShardManifest { /// A human-readable what-if summary line. pub fn summary(&self) -> String { - let hbs: Vec = self.shards.iter().map(|s| s.heartbeats).collect(); + let costs: Vec = self.shards.iter().map(|s| s.cost.value()).collect(); let nonempty: Vec = self .shards .iter() .filter(|s| !s.blocks.is_empty()) - .map(|s| s.heartbeats) + .map(|s| s.cost.value()) .collect(); - let max = hbs.iter().copied().max().unwrap_or(0); + let unit = self + .shards + .first() + .map(|s| s.cost.unit()) + .filter(|u| !u.is_empty()) + .unwrap_or("cost"); + let max = costs.iter().copied().max().unwrap_or(0); let min = nonempty.iter().copied().min().unwrap_or(0); - let total: u128 = hbs.iter().map(|&h| u128::from(h)).sum(); + let total: u128 = costs.iter().map(|&h| u128::from(h)).sum(); let mean = if self.shards.is_empty() { 0 } else { @@ -1384,7 +1464,7 @@ impl ShardManifest { let max_cross = self.shards.iter().map(|s| s.cross_ingress).max().unwrap_or(0); format!( - "shards={} (empty={}) heartbeats[min={} mean={} max={}] imbalance={:.2}x \ + "shards={} (empty={}) {unit}[min={} mean={} max={}] imbalance={:.2}x \ cross_ingress_total={} max_shard_cross={}", self.shards.len(), empty, @@ -1411,7 +1491,8 @@ impl ShardManifest { }; for sh in &self.shards { out.extend_from_slice(&sh.id.to_le_bytes()); - out.extend_from_slice(&sh.heartbeats.to_le_bytes()); + out.push(sh.cost.tag()); + out.extend_from_slice(&sh.cost.value().to_le_bytes()); out.extend_from_slice(&sh.own_size.to_le_bytes()); out.extend_from_slice(&sh.cross_ingress.to_le_bytes()); match &sh.assumption_root { @@ -1440,7 +1521,15 @@ impl ShardManifest { /// Deserialize from the `.ixes` binary format. pub fn from_bytes(bytes: &[u8]) -> Result { let mut c = Cur { buf: bytes, pos: 0 }; - if c.take(8)? != SHARD_MAGIC { + let magic = c.take(8)?; + if magic != SHARD_MAGIC { + if magic.starts_with(b"IXES") { + return Err(format!( + "unsupported .ixes format version {} (expected {}) — regenerate \ + the manifest with the current `ix shard`/`ix shard scan`", + magic[7], SHARD_MAGIC[7] + )); + } return Err("not an .ixes file (bad magic)".into()); } let total_cross_ingress = c.u128()?; @@ -1448,7 +1537,7 @@ impl ShardManifest { let mut shards = Vec::with_capacity(num_shards); for _ in 0..num_shards { let id = c.u32()?; - let heartbeats = c.u64()?; + let cost = ShardCost::from_tag(c.u8()?, c.u64()?)?; let own_size = c.u64()?; let cross_ingress = c.u64()?; let assumption_root = if c.u8()? == 1 { Some(c.addr()?) } else { None }; @@ -1457,7 +1546,7 @@ impl ShardManifest { shards.push(ShardInfo { id, blocks, - heartbeats, + cost, own_size, foreign_blocks, cross_ingress, @@ -1497,8 +1586,10 @@ impl ShardManifest { } } -/// Magic bytes at the head of every `.ixes` file. -const SHARD_MAGIC: &[u8; 8] = b"IXES\0\0\0\0"; +/// Magic bytes at the head of every `.ixes` file; the final byte is the +/// format version (bumped to 2 when per-shard tagged costs replaced the +/// bare heartbeats field). +const SHARD_MAGIC: &[u8; 8] = b"IXES\0\0\0\x02"; /// Minimal little-endian cursor for manifest decoding. struct Cur<'a> { @@ -1580,7 +1671,7 @@ pub fn shard_esp( let max_block_hb = profile.blocks().iter().map(|b| b.heartbeats).max().unwrap_or(0); let max_shard_hb = - manifest.shards.iter().map(|s| s.heartbeats).max().unwrap_or(0); + manifest.shards.iter().map(|s| s.cost.value()).max().unwrap_or(0); let floored = num_shards > 1 && max_shard_hb <= max_block_hb.saturating_mul(11) / 10; let note = if floored { @@ -1762,6 +1853,121 @@ pub fn shard_prove_secs(steps: u64) -> f64 { PROVE_SETUP_SECS + PROVE_SECS_PER_BCOST * (steps as f64 / 1e9) } +/// `x·log₂(x+2)` — the feature form of the Aiur cost model. Aiur's prover work +/// is a sum of `width·height·log₂(height)` FFTs over its circuits, and each +/// dominant circuit family's height tracks one kernel counter, so a counter's +/// cost contribution is super-linear with exactly this shape. The `+2` keeps +/// the log positive at zero. +#[allow(clippy::cast_precision_loss)] // counters are far below 2^53 +fn nlogn(x: u64) -> f64 { + let x = x as f64; + x * (x + 2.0).log2() +} + +/// LEGACY counter-model constants: advisory pricing for `ix profile` +/// sweep/leaderboards only. Superseded for shard sizing by the measured +/// scan + analytic peak-prove-RAM model; do not use these to size shards. +/// +/// Calibrated two-stage Aiur cost model. +/// +/// **Stage 1** predicts a run's total FFT cost (the prover's actual work +/// unit: `Σ width·2^⌈log h⌉·log h` over circuits) from the profile +/// counters of the run's owned blocks plus its faulted-set bytes. `def_eq` +/// is a load-bearing feature: definitional-equality-dense checks drive +/// trace volume that bytes/subst alone under-predict. +/// +/// **Stage 2** maps FFT cost linearly to prove wall seconds and peak host +/// RAM — physically grounded (committed LDE volume is proportional to FFT +/// work) and measured tight (≤10% on RAM, ≤12% on wall). +/// +/// Fit 2026-08-03 on this kernel (addr-first, lazy fault-in): stage 1 +/// against the exact per-shard FFT costs of all 34 Init shards at a 250 +/// GiB pack (execute-mode stats dumps; MAPE 5.0%, worst under −18.5%), +/// stage 2 against 13 measured shard proves spanning 21–81 BFFT (RSS +/// max |err| 9.9%, wall 12.2%). Composed end-to-end on the proved set: +/// RAM within 12.4%. The prove base term includes env load + system +/// setup, matching what the batch driver schedules. +pub const AIUR_FFT_BASE: f64 = 2.599e9; +pub const AIUR_FFT_PER_NLOGN_INGRESS_BYTE: f64 = 188.6; +pub const AIUR_FFT_PER_NLOGN_SUBST: f64 = 71.27; +pub const AIUR_FFT_PER_NLOGN_DEF_EQ: f64 = 6548.0; +pub const AIUR_PROVE_BASE_SECS: f64 = 6.18; +pub const AIUR_PROVE_SECS_PER_BFFT: f64 = 2.0425; +pub const AIUR_RAM_BASE_GIB: f64 = 13.59; +pub const AIUR_RAM_GIB_PER_BFFT: f64 = 2.3507; +/// Usable fraction of an Aiur host-RAM budget. Covers the composed model's +/// measured worst under-prediction (stage-1 tail −18.5% × stage-2 −8.5% ≈ +/// −25%, so predictions at cap stay under budget), with the remainder as +/// OS/variance margin. +pub const AIUR_RAM_USABLE_FRAC: f64 = 0.75; + +/// Stage 1: predicted total FFT cost for one run faulting in `bytes`, +/// with `subst` substitution-node visits and `def_eq` definitional +/// equality checks over its owned blocks. +pub fn aiur_shard_fft(bytes: u64, subst: u64, def_eq: u64) -> f64 { + AIUR_FFT_BASE + + AIUR_FFT_PER_NLOGN_INGRESS_BYTE * nlogn(bytes) + + AIUR_FFT_PER_NLOGN_SUBST * nlogn(subst) + + AIUR_FFT_PER_NLOGN_DEF_EQ * nlogn(def_eq) +} + +/// Stage 2: prove wall seconds for a run of `fft` total FFT cost. Feed a +/// measured FFT cost (an execute-mode stats dump) for an exact-height +/// prediction, or [`aiur_shard_fft`]'s estimate at plan time. +pub fn aiur_prove_secs_for_fft(fft: f64) -> f64 { + AIUR_PROVE_BASE_SECS + AIUR_PROVE_SECS_PER_BFFT * (fft / 1e9) +} + +/// Stage 2: peak prover host RAM (GiB) for a run of `fft` total FFT cost. +pub fn aiur_ram_gib_for_fft(fft: f64) -> f64 { + AIUR_RAM_BASE_GIB + AIUR_RAM_GIB_PER_BFFT * (fft / 1e9) +} + +/// Composed plan-time prediction: prove wall seconds from profile features. +pub fn aiur_prove_secs(bytes: u64, subst: u64, def_eq: u64) -> f64 { + aiur_prove_secs_for_fft(aiur_shard_fft(bytes, subst, def_eq)) +} + +/// Composed plan-time prediction: peak prover host RAM (GiB) from profile +/// features. +pub fn aiur_ram_gib(bytes: u64, subst: u64, def_eq: u64) -> f64 { + aiur_ram_gib_for_fft(aiur_shard_fft(bytes, subst, def_eq)) +} + +/// A block's marginal predicted Aiur prove time (seconds), `nlogn` taken at +/// block granularity and the per-run bases omitted. Slightly under-counts a +/// block's share inside a large run (whose `log` factor is bigger), which is +/// fine for its purpose: ranking blocks in the `ix profile` leaderboards. +pub fn aiur_block_prove_secs(b: &BlockEntry) -> f64 { + let fft = AIUR_FFT_PER_NLOGN_INGRESS_BYTE + * nlogn(u64::from(b.serialized_size)) + + AIUR_FFT_PER_NLOGN_SUBST * nlogn(b.subst) + + AIUR_FFT_PER_NLOGN_DEF_EQ * nlogn(b.def_eq); + AIUR_PROVE_SECS_PER_BFFT * (fft / 1e9) +} + +/// Calibrated Aiur **execute** (witness-generation, no prove) cost model — +/// fit on the same 34-shard Init sweep as stage 1 (wall MAPE 9.6%, RSS +/// 3.7%). Execute wall tracks `def_eq` alone; execute RSS adds the +/// faulted-byte term. +pub const AIUR_EXEC_BASE_SECS: f64 = 3.72; +pub const AIUR_EXEC_SECS_PER_NLOGN_DEF_EQ: f64 = 3.602e-6; +pub const AIUR_EXEC_RAM_BASE_GIB: f64 = 1.34; +pub const AIUR_EXEC_RAM_GIB_PER_NLOGN_INGRESS_BYTE: f64 = 3.008e-8; +pub const AIUR_EXEC_RAM_GIB_PER_NLOGN_DEF_EQ: f64 = 6.119e-7; + +/// Predicted Aiur execute time (seconds) for one run. +pub fn aiur_exec_secs(def_eq: u64) -> f64 { + AIUR_EXEC_BASE_SECS + AIUR_EXEC_SECS_PER_NLOGN_DEF_EQ * nlogn(def_eq) +} + +/// Predicted Aiur execute peak host RAM (GiB) for one run. +pub fn aiur_exec_ram_gib(bytes: u64, def_eq: u64) -> f64 { + AIUR_EXEC_RAM_BASE_GIB + + AIUR_EXEC_RAM_GIB_PER_NLOGN_INGRESS_BYTE * nlogn(bytes) + + AIUR_EXEC_RAM_GIB_PER_NLOGN_DEF_EQ * nlogn(def_eq) +} + /// Whole-workload prove-time estimate over a partition's per-shard step counts. pub struct ProveEstimate { /// Σ predicted guest STEPS over all shards (incl. per-shard floor + ingress). @@ -1934,31 +2140,12 @@ pub fn partition_for_cycle_cap( }; } - // 1. Cut-coherent block order. A fine min-cut pre-partition's bisection tree, - // read in DFS order, lays tightly-coupled blocks contiguously so the packer - // keeps dependency overlap within a shard. Sized to ~PACK_PIECES_PER_CAP - // pieces per cap (bounded by the block count); when everything fits one cap - // this collapses to a trivial order. + // 1. Cut-coherent block order, sized to ~PACK_PIECES_PER_CAP pieces per cap. let total: u128 = profile.blocks().iter().map(|b| u128::from(block_step_cost(b))).sum(); let pieces = ((total.saturating_mul(PACK_PIECES_PER_CAP)) / u128::from(step_cap)) as usize; - let n_fine = pieces.clamp(1, nblocks); - let order: Vec = if n_fine < 2 { - (0..nblocks as u32).collect() - } else { - let h = Hypergraph::from_profile(profile); - let (fine_of, fine_tree) = h.partition_with_tree(n_fine, epsilon); - let mut leaf_order = Vec::with_capacity(n_fine); - dfs_leaf_order(&fine_tree, &mut leaf_order); - let mut rank = vec![0u32; n_fine]; - for (r, &sid) in leaf_order.iter().enumerate() { - rank[sid as usize] = r as u32; - } - let mut order: Vec = (0..nblocks as u32).collect(); - order.sort_by_key(|&b| (rank[fine_of[b as usize] as usize], b)); - order - }; + let order = cut_coherent_order(profile, pieces, epsilon); // 2. Greedy next-fit packing to the cap, with live cross-ingress accounting. // A shard accumulates members until adding the next block would push its @@ -2055,6 +2242,35 @@ pub fn partition_for_cycle_cap( } } +/// A cut-coherent block order for greedy cap-packing: a fine min-cut +/// pre-partition's bisection tree, read in DFS order, lays tightly-coupled +/// blocks contiguously so a next-fit packer keeps dependency overlap within a +/// shard. `pieces` is the requested fine-partition size (~[`PACK_PIECES_PER_CAP`] +/// per cap, in the caller's cost unit), bounded by the block count; when +/// everything fits one cap this collapses to the trivial order. +pub fn cut_coherent_order( + profile: &BlockProfile, + pieces: usize, + epsilon: f64, +) -> Vec { + let nblocks = profile.num_blocks(); + let n_fine = pieces.clamp(1, nblocks); + if n_fine < 2 { + return (0..nblocks as u32).collect(); + } + let h = Hypergraph::from_profile(profile); + let (fine_of, fine_tree) = h.partition_with_tree(n_fine, epsilon); + let mut leaf_order = Vec::with_capacity(n_fine); + dfs_leaf_order(&fine_tree, &mut leaf_order); + let mut rank = vec![0u32; n_fine]; + for (r, &sid) in leaf_order.iter().enumerate() { + rank[sid as usize] = r as u32; + } + let mut order: Vec = (0..nblocks as u32).collect(); + order.sort_by_key(|&b| (rank[fine_of[b as usize] as usize], b)); + order +} + /// Collect the leaf shard ids of an [`AggNode`] in left-to-right DFS order. fn dfs_leaf_order(node: &AggNode, out: &mut Vec) { match node { @@ -2067,7 +2283,7 @@ fn dfs_leaf_order(node: &AggNode, out: &mut Vec) { } /// A balanced binary [`AggNode`] over the contiguous shard-id range `lo..hi`. -fn balanced_agg_tree(lo: u32, hi: u32) -> AggNode { +pub fn balanced_agg_tree(lo: u32, hi: u32) -> AggNode { debug_assert!(hi > lo); if hi - lo <= 1 { AggNode::Leaf(lo) @@ -2156,8 +2372,8 @@ mod tests { let m = ShardManifest::build(&p, &shard_of, 2); assert_eq!(m.shards.len(), 2); // Each cluster has 3 blocks × 100 heartbeats = 300; perfectly balanced. - assert_eq!(m.shards[0].heartbeats, 300); - assert_eq!(m.shards[1].heartbeats, 300); + assert_eq!(m.shards[0].cost, ShardCost::ProfileHeartbeats(300)); + assert_eq!(m.shards[1].cost, ShardCost::ProfileHeartbeats(300)); } #[test] @@ -2183,7 +2399,7 @@ mod tests { assert_eq!(m.total_cross_ingress, 0); // Each non-empty shard should hold exactly one cluster (4×100). for s in &m.shards { - assert_eq!(s.heartbeats, 400); + assert_eq!(s.cost, ShardCost::ProfileHeartbeats(400)); } } @@ -2313,6 +2529,41 @@ mod tests { assert!(plan.infeasible_atomic_floor, "oversized atomic block must flag"); } + #[test] + fn aiur_model_monotone() { + // The packer's greedy cap test is only sound if the model never decreases + // when a shard grows in any aggregate. + let base = aiur_ram_gib(1_000_000, 10_000, 10_000); + assert!(aiur_ram_gib(2_000_000, 10_000, 10_000) > base); + assert!(aiur_ram_gib(1_000_000, 20_000, 10_000) > base); + assert!(aiur_ram_gib(1_000_000, 10_000, 20_000) > base); + let p = aiur_prove_secs(1_000_000, 100_000, 10_000); + assert!(aiur_prove_secs(2_000_000, 100_000, 10_000) > p); + assert!(aiur_prove_secs(1_000_000, 200_000, 10_000) > p); + assert!(aiur_prove_secs(1_000_000, 100_000, 20_000) > p); + } + + /// Attach a reference graph given per-block (sorted, deduped) ref lists. + fn with_refs(mut p: BlockProfile, adj: &[Vec]) -> BlockProfile { + p.set_ref_graph(adj); + p + } + + #[test] + fn ref_graph_roundtrips_through_ixprof() { + let mut b = ProfileBuilder::new(); + for i in 1..=3u8 { + b.block(addr(i), 1, 10, 1, ops(2)); + } + let p = with_refs(b.finish(), &[vec![1, 2], vec![2], vec![]]); + let q = BlockProfile::from_bytes(&p.to_bytes()).unwrap(); + assert!(q.has_ref_graph()); + assert_eq!(q.refs(0), &[1, 2]); + assert_eq!(q.refs(1), &[2]); + assert_eq!(q.refs(2), &[] as &[u32]); + assert_eq!(p, q); + } + #[test] fn shard_esp_file_roundtrip() { // Exercise the CLI's shard path: write a .ixprof, run shard_esp, read back diff --git a/crates/kernel/src/subst.rs b/crates/kernel/src/subst.rs index 9779c76b..13a0423b 100644 --- a/crates/kernel/src/subst.rs +++ b/crates/kernel/src/subst.rs @@ -828,6 +828,15 @@ pub fn instantiate_rev( if fvars.is_empty() || body.lbr() == 0 { return body.clone(); } + // Profiler: the substitution-context half of the unique-work key — a + // fold of the fvar identities, set once per top-level call (the + // recursion below never re-enters subst, so the slot stays valid). + #[cfg(not(target_os = "zkvm"))] + crate::profile::set_subst_ctx( + fvars.iter().fold(0xcbf2_9ce4_8422_2325_u64, |h, f| { + (h ^ f.hash_key()).wrapping_mul(0x0000_0100_0000_01b3) + }), + ); // Borrow the dedicated `subst_scratch` (same allocation reuse trick as // `subst`/`simul_subst`). `instantiate_rev_cached` does not call back // into subst/simul_subst/lift, so the scratch is safe to share across @@ -847,9 +856,12 @@ fn instantiate_rev_cached( cache: &mut FxHashMap<(Addr, u64), KExpr>, ) -> KExpr { // Profiler: count every substitution-node visit — the work-volume feature - // that `heartbeats` (step count) misses. Records out of circuit; the bump - // compiles out on the zkvm target. + // that `heartbeats` (step count) misses — and its memo-deduplicated + // counterpart keyed on (expr, depth, substitution context). Records out + // of circuit; both bumps compile out on the zkvm target. crate::profile::bump_subst_nodes(); + #[cfg(not(target_os = "zkvm"))] + crate::profile::bump_subst_unique(body.hash_key(), depth); // No loose bvars at or below `depth` means nothing to instantiate at // this subtree. if body.lbr() <= depth { diff --git a/crates/kernel/src/tc.rs b/crates/kernel/src/tc.rs index 252798cc..4f12b704 100644 --- a/crates/kernel/src/tc.rs +++ b/crates/kernel/src/tc.rs @@ -172,6 +172,11 @@ pub struct TypeChecker<'a, M: KernelMode> { /// Addresses of constants whose bodies were delta-unfolded during the current /// constant's check. Drained per constant by `record_current_fuel_used`. pub(crate) delta_targets: FxHashSet
, + /// Every constant CONSULTED while checking the current one, whether its + /// body was unfolded or only its type read — the measured ingress set of + /// a lazy checker. Drained per constant by `record_current_fuel_used` + /// into the profile sink, which persists it as the `.ixprof` touch graph. + pub(crate) touched: FxHashSet
, /// Gated miss sampler for fuel-exhaustion diagnostics. Populated only when /// `IX_HOT_MISSES=1`, keyed by a compact phase/head/lbr shape. hot_misses: FxHashMap, @@ -222,6 +227,7 @@ impl<'a, M: KernelMode> TypeChecker<'a, M> { debug_label: None, cur_const: None, delta_targets: FxHashSet::default(), + touched: FxHashSet::default(), hot_misses: FxHashMap::default(), ctx_addr_cache: FxHashMap::default(), lctx: super::lctx::LocalContext::new(), @@ -252,6 +258,9 @@ impl<'a, M: KernelMode> TypeChecker<'a, M> { &mut self, id: &KId, ) -> Result>, TcError> { + if self.env.profile_sink.is_some() { + self.touched.insert(id.addr.clone()); + } if let Some(c) = self.env.get(id) { return Ok(Some(c)); } @@ -914,13 +923,19 @@ impl<'a, M: KernelMode> TypeChecker<'a, M> { // delta-unfold edges. `delta_targets` is always drained (even when // `cur_const` is unset) so producers never leak into the next constant. if self.env.profile_sink.is_some() { + let touched = std::mem::take(&mut self.touched); + if std::env::var_os("IX_TOUCH_STATS").is_some() + && let Some(a) = self.cur_const.as_ref() + { + eprintln!("[touch] {} {}", a.hex(), touched.len()); + } let producers = std::mem::take(&mut self.delta_targets); // Always drain the op counters (even when `cur_const` is unset) so they // never leak into the next constant, mirroring `delta_targets`. let ops = crate::profile::take_op_counts(); if let Some(addr) = self.cur_const.take() { let sink = self.env.profile_sink.as_mut().unwrap(); - sink.record(addr, used, producers, ops); + sink.record(addr, used, producers, touched, ops); } } } diff --git a/docs/heavy-constant-sharding.md b/docs/heavy-constant-sharding.md new file mode 100644 index 00000000..836d0468 --- /dev/null +++ b/docs/heavy-constant-sharding.md @@ -0,0 +1,115 @@ +# Heavy constants: closure-shard proving + +Some constants cannot be proved as a single claim on any reasonable +machine. `Std.Tactic.BVDecide.BVExpr.bitblast.goCache_Inv_of_Inv._mutual` +(the ~18B-step bitblast mutual block, 5659 constants in closure) is the +canonical example: its standalone prove measured **473.5 GiB peak** on a +495 GB box and OOM'd the 128 GB CI runner outright. This document +describes why, and the closure-shard strategy that makes such constants +provable at any RAM budget — measured at **no cost in wall clock or +total compute**. + +## Why a standalone prove is the most expensive shape + +A standalone claim (`Claim.check addr none`) carries **no assumptions**: +the kernel must re-derive the constant's entire transitive dependency +spine in-circuit — every definitional unfolding and def-eq down to the +`Nat` primitives. Nothing is shared with any other proof. + +The env partition proves the same constant differently. A shard's +`CheckEnv` claim has a **thin frontier**: dependencies outside the shard +are *assumed* — named by content address and committed in a Merkle tree +(a few blake3 rows each) — not re-checked. Other shards' proofs cover +them, and the composed verdict glues the partition through the coverage +gate (every block owned exactly once, assumption roots matching). + +Measured consequence: bitblast's standalone closure record costs more +than an *entire InitStd env shard* containing bitblast **plus 6655 +neighboring blocks** (473.5 GiB / 5:43 standalone, vs 348.1 GiB / 3:13 +for the whole shard). Hashing a frontier entry costs a few circuit rows; +re-deriving its body costs its whole checking cone. + +## The strategy: shard the closure itself + +Make the closure its own mini-env and partition *it*: + +```bash +ix shard extract env.ixe --consts Foo.bar --out foo.ixe # closure → standalone env +ix shard foo.ixe --max-ram 100 --out foo.ixes # measured union-pricing cut +ix prove --ixe foo.ixe --ixes foo.ixes # prove ALL shards +``` + +The partition's thin frontiers are *internal to the closure*, so proving +every shard yields the same unconditional verdict as the standalone +claim — at per-shard RAM chosen by `--max-ram`. The all-shards prove +shares one env load across shards, verifies each proof and binds it to +its reconstructed claim, and persists progress in +`~/.ix/cache/shard-proofs/` keyed by claim digest — a killed run +resumes, and a repacked manifest re-proves only shards that changed. +`--consts` takes several names (their union closure extracts together); +a mutual-block member extracts its whole block; `--max-ram` defaults to +detected system RAM. + +## Measured: bitblast, three ways (495 GB box, 64 cores) + +| Prove | Wall | Total compute (exec+STARK) | Peak RAM | Proofs | +| ----------------- | ---------- | -------------------------- | ------------- | ----------- | +| Standalone | 5:43 | 341.8 s | **473.5 GiB** | 1 × 23.5 MB | +| 2 shards (@480) | **4:46** | **284.0 s** | 325.8 GiB | 2 × ~24 MB | +| 9 shards (@108) | 5:40 | 333.5 s | **83.2 GiB** | 9 × ~22 MB | + +| Execute | Total kernel time | Peak RAM | +| ----------------- | ----------------- | -------- | +| Standalone | 70.4 s | 25.1 GiB | +| 2 shards (@480) | 69.6 s | 19.0 GiB | +| 9 shards (@108) | 70.9 s | 8.0 GiB | + +Two effects cancel to make splitting free (or better): + +- **Frontier tax**: each shard re-derives frontier-adjacent context its + claim assumes was elsewhere — total work rises with shard count. +- **Padding savings**: trace heights commit at `next_power_of_two`, so + several short records often pad to lower power-of-two boundaries than + one tall record — total work *falls* with shard count. + +At 9 shards these wash (−2% net); at 2 shards padding wins outright +(−17% net, and less wall than standalone). RAM tracks the budget almost +linearly: 473 → 326 → 83 GiB. + +Serial wall is the pessimistic case. The shards are independent proofs: +a fleet of small boxes (or `ix prove --jobs N` on one big box) proves +them concurrently, collapsing wall toward the slowest single shard +(~42 s here). The standalone prove has no such option — it is one +indivisible 473 GiB job. Closure-sharding converts a RAM-bound serial +prove into parallelizable units. + +## CI integration + +`ix bench run --backend aiur` routes **heavy-tier** `Vectors.csv` +constants through this pipeline automatically (`cutAiurClosureShards`: +extract → measured scan at the watchdog ceiling → one +`check`/`prove --shard K` spawn per shard). Each shard reports a +`/shard-K` sub-row; the parent row aggregates (summed time — the +serial cost — max peak-rss, shard count, the manifest's total measured +fft) and lands only when every shard is green. Light-tier constants keep +the single-leaf spawn: cheap closures gain nothing from partition +overhead. + +At the CI runner class (measured at a 108 GiB ceiling) bitblast proves +in 9 shards, 83.2 GiB peak, ~5.7 min serial — inside a 128 GB runner +with margin. Note `(bitblast, aiur, prove)` remains in +`benchExclusions` until a scheduled CI row is wanted; an explicit +`--consts` request always runs. + +## Caveats + +- **N proofs, not one.** The composed verdict is sound, but "a proof of + Foo.bar" is a set of shard proofs until recursive aggregation lands + (9 × ~22 MB vs 1 × 23.5 MB at the fine split). +- The bench cache (`aiurshards-/.{ixe,ixes}`) is keyed by + constant only, not budget: re-cutting at a different ceiling requires + deleting the stale `.ixes` (the extracted `.ixe` is budget-independent + and stays). The raw CLI has no such trap — outputs are named + explicitly. +- A closure that fits the budget in one shard degenerates to exactly the + standalone prove; there is no penalty for trying the pipeline first.