From 6c8ac56dc7bbecd8c73ad05d67015141fc8419f6 Mon Sep 17 00:00:00 2001 From: Ammar Arif Date: Tue, 2 Jun 2026 16:37:16 -0500 Subject: [PATCH 1/4] fix(persistent-tee): settle sequentially with on-chain gating and retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The settlement loop dropped a proof on any transient error (estimate_fee / send / watch_tx) via `continue` — with no retry and no check against the Piltover contract's on-chain block before submitting. Fee estimation runs against `latest`, which lags `pre_confirmed`, so under load `update_state` for block N+1 is submitted before `latest` reflects block N and the contract rejects it with "State: invalid block number". The proof is then dropped, and because every later block's `prev_block_number` no longer matches the on-chain state, the whole pipeline wedges permanently: settlement stops while the prover/ingestor race ahead, and it never recovers without a restart. Settle strictly in order instead: only submit `update_state` for a block once `get_state().block == proof.prev_block_number`; otherwise wait (a prior settlement is still landing, or `latest` lags `pre_confirmed`). Retry transient failures on the same proof rather than skipping ahead, and skip blocks already settled on-chain (idempotent across restarts). A single transient failure no longer wedges the backend. Co-Authored-By: Claude Opus 4.8 --- bin/persistent-tee/src/settlement.rs | 130 ++++++++++++++++++++------- 1 file changed, 97 insertions(+), 33 deletions(-) diff --git a/bin/persistent-tee/src/settlement.rs b/bin/persistent-tee/src/settlement.rs index 4d53947..1c66867 100644 --- a/bin/persistent-tee/src/settlement.rs +++ b/bin/persistent-tee/src/settlement.rs @@ -15,7 +15,7 @@ use starknet::{ signers::{LocalWallet, SigningKey}, }; use tokio::sync::mpsc::{Receiver, Sender}; -use tracing::{debug, error}; +use tracing::{debug, error, warn}; use url::Url; use saya_core::{ @@ -248,8 +248,17 @@ impl TeePiltoverSettlementBackend { } } + /// Sleep one polling interval; returns `true` if shutdown was requested while + /// waiting, so retry loops can stop promptly instead of hanging. + async fn cooldown(&self) -> bool { + tokio::select! { + _ = self.finish_handle.shutdown_requested() => true, + _ = tokio::time::sleep(POLLING_INTERVAL) => false, + } + } + async fn run(mut self) { - loop { + 'outer: loop { let proof = tokio::select! { _ = self.finish_handle.shutdown_requested() => break, p = self.proof_channel.recv() => match p { @@ -261,6 +270,12 @@ impl TeePiltoverSettlementBackend { }, }; + // Calldata is a pure function of the proof; a build failure means a + // malformed proof and retrying can't help, so skip it. The settlement + // errors below are the opposite — transient — so we RETRY the same proof + // and never drop it. Dropping a proof leaves a permanent gap: every later + // block's `prev_block_number` then mismatches the on-chain state and the + // contract rejects it forever ("State: invalid block number"). let calldata = match build_tee_calldata(&proof, self.mock_prove) { Ok(c) => c, Err(e) => { @@ -273,46 +288,95 @@ impl TeePiltoverSettlementBackend { } }; - let call = Call { - to: self.piltover_address, - selector: selector!("update_state"), - calldata, - }; + // Settle strictly in order. Only submit `update_state` once the Piltover + // contract's on-chain block equals this proof's parent; otherwise wait — + // a prior settlement is still landing, or `latest` lags `pre_confirmed` + // (fee estimation runs against `latest`, so submitting early reverts with + // "invalid block number"). Retry transient failures rather than advancing + // to the next proof. A `None` result means the block was already settled + // on-chain (e.g. resumed after a restart), so we just advance the cursor. + let tx_hash: Option = loop { + if self.finish_handle.is_shutdown_requested() { + break 'outer; + } - let execution = self.account.execute_v3(vec![call]); + let onchain = match self.get_piltover_block_number().await { + Ok(b) => b, + Err(e) => { + warn!( + "Failed to read Piltover block for {}: {}; retrying", + proof.block_number.to_hex_string(), + e + ); + if self.cooldown().await { + break 'outer; + } + continue; + } + }; - let _fees = match execution.estimate_fee().await { - Ok(f) => f, - Err(e) => { - error!( - "Fee estimation failed for block {}: {}", - proof.block_number.to_hex_string(), - e + if onchain >= proof.block_number { + debug!( + "Block {} already settled on-chain; advancing cursor", + proof.block_number.to_hex_string() ); - continue; + break None; } - }; - let transaction = match execution.send().await { - Ok(t) => t, - Err(e) => { - error!( - "Settlement transaction failed for block {}: {}", - proof.block_number, e - ); + if onchain != proof.prev_block_number { + // Parent not yet on-chain — wait and recheck; do NOT skip ahead. + if self.cooldown().await { + break 'outer; + } continue; } - }; - match self.watch_tx(transaction.transaction_hash).await { - Ok(()) => {} - Err(e) => { - error!( - "Settlement tx confirmation failed for block {}: {}", - proof.block_number, e + let call = Call { + to: self.piltover_address, + selector: selector!("update_state"), + calldata: calldata.clone(), + }; + let execution = self.account.execute_v3(vec![call]); + + if let Err(e) = execution.estimate_fee().await { + warn!( + "Fee estimation failed for block {}: {}; retrying", + proof.block_number.to_hex_string(), + e ); + if self.cooldown().await { + break 'outer; + } continue; } - } + let transaction = match execution.send().await { + Ok(t) => t, + Err(e) => { + warn!( + "Settlement transaction failed for block {}: {}; retrying", + proof.block_number.to_hex_string(), + e + ); + if self.cooldown().await { + break 'outer; + } + continue; + } + }; + match self.watch_tx(transaction.transaction_hash).await { + Ok(()) => break Some(transaction.transaction_hash), + Err(e) => { + warn!( + "Settlement tx confirmation failed for block {}: {}; retrying", + proof.block_number.to_hex_string(), + e + ); + if self.cooldown().await { + break 'outer; + } + continue; + } + } + }; let new_cursor = SettlementCursor { block_number: u64::try_from(proof.block_number).unwrap_or_else(|_| { @@ -321,7 +385,7 @@ impl TeePiltoverSettlementBackend { proof.block_number.to_hex_string() ) }), - transaction_hash: transaction.transaction_hash, + transaction_hash: tx_hash.unwrap_or(Felt::ZERO), }; tokio::select! { From 3042f2773722890fb475fb5d75f9a6beeed5637f Mon Sep 17 00:00:00 2001 From: Ammar Arif Date: Wed, 3 Jun 2026 10:48:59 -0500 Subject: [PATCH 2/4] test(persistent-tee): unit-test the settlement ordering gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the submit/wait/skip decision into `settlement_action()` and cover it with unit tests — a pure refactor, no behavior change. The key case is the regression for the wedge: when the prover/ingestor race ahead of the chain, a proof whose parent isn't on-chain yet must WAIT, not be submitted out of order (which reverts with "State: invalid block number" and, under the old drop-on-error loop, cascaded into a permanent stall). Also covers genesis, the submittable tip, and skipping already-settled blocks (restart idempotency). Co-Authored-By: Claude Opus 4.8 --- bin/persistent-tee/src/settlement.rs | 120 ++++++++++++++++++++++++--- 1 file changed, 108 insertions(+), 12 deletions(-) diff --git a/bin/persistent-tee/src/settlement.rs b/bin/persistent-tee/src/settlement.rs index 1c66867..d63a995 100644 --- a/bin/persistent-tee/src/settlement.rs +++ b/bin/persistent-tee/src/settlement.rs @@ -117,6 +117,36 @@ fn build_tee_calldata(proof: &TeeProof, mock_prove: bool) -> Result> { ))) } +/// What to do with a proof, given the Piltover contract's current on-chain block. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum SettlementAction { + /// Already settled on-chain (e.g. resumed after a restart, or a re-proved block) + /// — skip it and just advance the local cursor. + AlreadySettled, + /// The chain is exactly at this proof's parent — safe to submit `update_state`. + Submit, + /// The chain hasn't reached this proof's parent yet (a prior settlement is still + /// landing, or `latest` lags `pre_confirmed`) — wait and recheck; never skip. + WaitForParent, +} + +/// Decide whether to submit, wait, or skip a proof. +/// +/// Submitting only when the chain is exactly at the proof's parent keeps settlement +/// strictly in order: it never sends an `update_state` the contract would reject with +/// "invalid block number", and never skips a gap. Submitting out of order — or +/// dropping the failed block and moving on — is what wedged the pipeline; see the +/// unit tests below. +fn settlement_action(onchain_block: Felt, proof_prev: Felt, proof_block: Felt) -> SettlementAction { + if onchain_block >= proof_block { + SettlementAction::AlreadySettled + } else if onchain_block == proof_prev { + SettlementAction::Submit + } else { + SettlementAction::WaitForParent + } +} + /// Settlement backend that submits TEE proofs to the Piltover contract via `update_state`. #[derive(Debug)] pub struct TeePiltoverSettlementBackend { @@ -315,19 +345,22 @@ impl TeePiltoverSettlementBackend { } }; - if onchain >= proof.block_number { - debug!( - "Block {} already settled on-chain; advancing cursor", - proof.block_number.to_hex_string() - ); - break None; - } - if onchain != proof.prev_block_number { - // Parent not yet on-chain — wait and recheck; do NOT skip ahead. - if self.cooldown().await { - break 'outer; + match settlement_action(onchain, proof.prev_block_number, proof.block_number) { + SettlementAction::AlreadySettled => { + debug!( + "Block {} already settled on-chain; advancing cursor", + proof.block_number.to_hex_string() + ); + break None; } - continue; + SettlementAction::WaitForParent => { + // Parent not yet on-chain — wait and recheck; do NOT skip ahead. + if self.cooldown().await { + break 'outer; + } + continue; + } + SettlementAction::Submit => {} } let call = Call { @@ -414,3 +447,66 @@ impl Daemon for TeePiltoverSettlementBackend { tokio::spawn(self.run()); } } + +#[cfg(test)] +mod tests { + use super::{settlement_action, SettlementAction}; + use starknet::core::types::Felt; + + fn f(n: u64) -> Felt { + Felt::from(n) + } + + #[test] + fn submits_when_chain_is_at_the_proofs_parent() { + // Chain settled up to block 5; the next proof (block 6, parent 5) is the only + // one that may be submitted. + assert_eq!( + settlement_action(f(5), f(5), f(6)), + SettlementAction::Submit + ); + } + + #[test] + fn genesis_first_block_is_submittable() { + // Fresh chain at genesis (block 0): the first proof settles block 1 (parent 0). + assert_eq!( + settlement_action(f(0), f(0), f(1)), + SettlementAction::Submit + ); + } + + #[test] + fn waits_for_parent_instead_of_settling_out_of_order() { + // Regression for the settlement wedge: the prover/ingestor race ahead while the + // chain is stuck at block 5, offering block 10 (parent 9). Submitting it would + // revert with "State: invalid block number"; the old code then dropped the + // proof and every later block's parent mismatched, cascading into a permanent + // stall. The backend must WAIT for the parent — never submit out of order. + assert_eq!( + settlement_action(f(5), f(9), f(10)), + SettlementAction::WaitForParent + ); + // Even one block ahead must wait: `latest` lags `pre_confirmed`, so the parent + // may not be visible to fee estimation yet. + assert_eq!( + settlement_action(f(5), f(6), f(7)), + SettlementAction::WaitForParent + ); + } + + #[test] + fn skips_blocks_already_settled_on_chain() { + // Resumed after a restart with a stale local cursor: the chain is at 8, but a + // re-proved block 6 arrives — skip it (idempotent), don't re-settle. + assert_eq!( + settlement_action(f(8), f(5), f(6)), + SettlementAction::AlreadySettled + ); + // The proof's own block already being on-chain counts as settled too. + assert_eq!( + settlement_action(f(6), f(5), f(6)), + SettlementAction::AlreadySettled + ); + } +} From 3ec947088f22fdb3231549eb68210f58ab46e990 Mon Sep 17 00:00:00 2001 From: Ammar Arif Date: Wed, 3 Jun 2026 11:21:44 -0500 Subject: [PATCH 3/4] fix(persistent-tee): special-case the fresh-Piltover -1 block in the gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A freshly deployed Piltover reports its block number as -1 (felt PRIME-1 = `Felt::MAX`) before it has settled any block; the first proof is block 0 with parent -1. The settlement gate compared block numbers numerically, so that sentinel looked `>=` every block and made them all appear "already settled" — a fresh chain never started settling, it skipped every block. Handle the sentinel explicitly in `settlement_action`: at the fresh value, the only submittable proof is the genesis block (parent also -1); everything else waits. Covered by a test. Co-Authored-By: Claude Opus 4.8 --- bin/persistent-tee/src/settlement.rs | 37 +++++++++++++++++++++++++--- 1 file changed, 34 insertions(+), 3 deletions(-) diff --git a/bin/persistent-tee/src/settlement.rs b/bin/persistent-tee/src/settlement.rs index d63a995..e6d8238 100644 --- a/bin/persistent-tee/src/settlement.rs +++ b/bin/persistent-tee/src/settlement.rs @@ -130,6 +130,11 @@ enum SettlementAction { WaitForParent, } +/// The block number a freshly deployed Piltover contract reports before it has settled +/// any block: `-1` in the field, i.e. `PRIME - 1` ([`Felt::MAX`]). The first proof a +/// chain ever submits is block 0, whose `prev_block_number` is also this sentinel. +const FRESH_PILTOVER_BLOCK: Felt = Felt::MAX; + /// Decide whether to submit, wait, or skip a proof. /// /// Submitting only when the chain is exactly at the proof's parent keeps settlement @@ -138,6 +143,19 @@ enum SettlementAction { /// dropping the failed block and moving on — is what wedged the pipeline; see the /// unit tests below. fn settlement_action(onchain_block: Felt, proof_prev: Felt, proof_block: Felt) -> SettlementAction { + // A fresh Piltover (block == the -1 sentinel) has settled nothing yet, so no block + // can be "already settled" and the only submittable proof is the genesis block + // (block 0, whose parent is also the sentinel). Handle it explicitly: comparing the + // sentinel numerically would make it look `>=` every block and skip them all, so a + // fresh chain would never start settling. + if onchain_block == FRESH_PILTOVER_BLOCK { + return if proof_prev == FRESH_PILTOVER_BLOCK { + SettlementAction::Submit + } else { + SettlementAction::WaitForParent + }; + } + if onchain_block >= proof_block { SettlementAction::AlreadySettled } else if onchain_block == proof_prev { @@ -457,6 +475,11 @@ mod tests { Felt::from(n) } + /// The Piltover "no block settled yet" sentinel: block_number == -1 (PRIME - 1). + fn genesis() -> Felt { + Felt::ZERO - Felt::ONE + } + #[test] fn submits_when_chain_is_at_the_proofs_parent() { // Chain settled up to block 5; the next proof (block 6, parent 5) is the only @@ -468,12 +491,20 @@ mod tests { } #[test] - fn genesis_first_block_is_submittable() { - // Fresh chain at genesis (block 0): the first proof settles block 1 (parent 0). + fn settles_the_genesis_block_from_the_minus_one_sentinel() { + // Regression: a fresh Piltover reports its block as -1 (felt PRIME-1). The first + // proof is block 0 with parent -1. Comparing felts numerically made the huge + // sentinel look "already settled" >= every block, so settlement never started + // (it skipped every block). Genesis must be ordered below block 0 and submitted. assert_eq!( - settlement_action(f(0), f(0), f(1)), + settlement_action(genesis(), genesis(), f(0)), SettlementAction::Submit ); + // From genesis, block 1 (parent 0) must still wait for block 0 first. + assert_eq!( + settlement_action(genesis(), f(0), f(1)), + SettlementAction::WaitForParent + ); } #[test] From a600bde9152bf6c2a074173bc0fbad9b9b3e39fb Mon Sep 17 00:00:00 2001 From: Ammar Arif Date: Wed, 3 Jun 2026 11:55:41 -0500 Subject: [PATCH 4/4] test(persistent-tee): integration-test the settlement loop MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extract the settlement loop into a generic `run_settlement` over a small `SettlementChain` trait (`onchain_block` + `submit`). The real `PiltoverChain` impl is unchanged behaviour — `run` just destructures its fields into it and delegates — so the loop can now be driven against a fake chain in-process. Adds integration tests for the three behaviours that broke or regressed: - settles a fresh chain from the -1 genesis sentinel, in order; - retries a transient submit failure on the SAME block instead of dropping it and skipping ahead (the wedge); - skips blocks already settled on-chain after a restart. Co-Authored-By: Claude Opus 4.8 --- bin/persistent-tee/src/settlement.rs | 565 ++++++++++++++++++--------- 1 file changed, 370 insertions(+), 195 deletions(-) diff --git a/bin/persistent-tee/src/settlement.rs b/bin/persistent-tee/src/settlement.rs index e6d8238..e2bdbeb 100644 --- a/bin/persistent-tee/src/settlement.rs +++ b/bin/persistent-tee/src/settlement.rs @@ -165,6 +165,220 @@ fn settlement_action(onchain_block: Felt, proof_prev: Felt, proof_block: Felt) - } } +/// The chain operations the settlement loop needs, abstracted so [`run_settlement`] +/// can be unit-tested without a live Starknet node. +trait SettlementChain { + /// The Piltover contract's current on-chain block (`get_state` block_number). + async fn onchain_block(&self) -> Result; + /// Submit `update_state` for `proof` (calldata pre-built) and wait for it to be + /// accepted; returns the settlement tx hash. + async fn submit(&self, proof: &TeeProof, calldata: Vec) -> Result; +} + +/// Read the Piltover contract's current block number (`get_state()[1]`). +async fn piltover_block_number( + provider: &Arc>, + piltover_address: Felt, +) -> Result { + let raw = provider + .call( + FunctionCall { + contract_address: piltover_address, + entry_point_selector: selector!("get_state"), + calldata: vec![], + }, + BlockId::Tag(BlockTag::Latest), + ) + .await?; + // AppchainState: [state_root, block_number, block_hash] — block_number is index 1. + raw.get(1) + .copied() + .ok_or_else(|| anyhow::anyhow!("get_state returned fewer than 2 felts")) +} + +/// Poll a tx until it is accepted (or reverted / errored). +async fn watch_tx(provider: &Arc>, tx_hash: Felt) -> Result<()> { + loop { + tokio::time::sleep(POLLING_INTERVAL).await; + match provider.get_transaction_receipt(tx_hash).await { + Ok(receipt) => match receipt.receipt { + TransactionReceipt::Invoke(r) => { + use starknet::core::types::ExecutionResult; + match r.execution_result { + ExecutionResult::Succeeded => return Ok(()), + ExecutionResult::Reverted { reason } => { + return Err(anyhow::anyhow!("Transaction reverted: {reason}")) + } + } + } + _ => return Ok(()), + }, + Err(starknet::providers::ProviderError::StarknetError( + starknet::core::types::StarknetError::TransactionHashNotFound, + )) => continue, + Err(e) => return Err(e.into()), + } + } +} + +/// Sleep one polling interval; returns `true` if shutdown was requested while waiting, +/// so retry loops can stop promptly instead of hanging. +async fn cooldown(finish: &FinishHandle, poll_interval: Duration) -> bool { + tokio::select! { + _ = finish.shutdown_requested() => true, + _ = tokio::time::sleep(poll_interval) => false, + } +} + +/// The real chain: submits `update_state` to a Piltover contract on Starknet. +struct PiltoverChain { + provider: Arc>, + account: SingleOwnerAccount>, LocalWallet>, + piltover_address: Felt, +} + +impl SettlementChain for PiltoverChain { + async fn onchain_block(&self) -> Result { + piltover_block_number(&self.provider, self.piltover_address).await + } + + async fn submit(&self, _proof: &TeeProof, calldata: Vec) -> Result { + let call = Call { + to: self.piltover_address, + selector: selector!("update_state"), + calldata, + }; + let execution = self.account.execute_v3(vec![call]); + execution.estimate_fee().await?; + let transaction = execution.send().await?; + watch_tx(&self.provider, transaction.transaction_hash).await?; + Ok(transaction.transaction_hash) + } +} + +/// The settlement loop: drains proofs and settles them on `chain`, strictly in order. +/// +/// Generic over [`SettlementChain`] so it can be tested with a fake chain. The two +/// invariants it must uphold (see the tests): (1) only submit a proof when the chain +/// is at its parent, waiting otherwise; (2) RETRY a transient submit failure on the +/// same proof — never drop it, since a dropped proof permanently wedges every later +/// block. Already-settled proofs (chain ahead of the proof) are skipped. +async fn run_settlement( + chain: &C, + mut proof_channel: Receiver, + cursor_channel: Sender, + finish_handle: FinishHandle, + poll_interval: Duration, + mock_prove: bool, +) { + 'outer: loop { + let proof = tokio::select! { + _ = finish_handle.shutdown_requested() => break, + p = proof_channel.recv() => match p { + Some(p) => p, + None => { + debug!("Proof channel closed, shutting down"); + break; + } + }, + }; + + // Calldata is a pure function of the proof; a build failure means a malformed + // proof and retrying can't help, so skip it. The settlement errors below are + // the opposite — transient — so we RETRY the same proof and never drop it. + // Dropping a proof leaves a permanent gap: every later block's + // `prev_block_number` then mismatches the on-chain state and the contract + // rejects it forever ("State: invalid block number"). + let calldata = match build_tee_calldata(&proof, mock_prove) { + Ok(c) => c, + Err(e) => { + error!( + "Failed to build TEE calldata for block {}: {}", + proof.block_number.to_hex_string(), + e + ); + continue; + } + }; + + // Settle strictly in order. Only submit once the chain's on-chain block equals + // this proof's parent; otherwise wait. Retry transient failures rather than + // advancing to the next proof. A `None` result means the block was already + // settled on-chain (e.g. resumed after a restart), so we just advance the cursor. + let tx_hash: Option = loop { + if finish_handle.is_shutdown_requested() { + break 'outer; + } + + let onchain = match chain.onchain_block().await { + Ok(b) => b, + Err(e) => { + warn!( + "Failed to read Piltover block for {}: {}; retrying", + proof.block_number.to_hex_string(), + e + ); + if cooldown(&finish_handle, poll_interval).await { + break 'outer; + } + continue; + } + }; + + match settlement_action(onchain, proof.prev_block_number, proof.block_number) { + SettlementAction::AlreadySettled => { + debug!( + "Block {} already settled on-chain; advancing cursor", + proof.block_number.to_hex_string() + ); + break None; + } + SettlementAction::WaitForParent => { + // Parent not yet on-chain — wait and recheck; do NOT skip ahead. + if cooldown(&finish_handle, poll_interval).await { + break 'outer; + } + continue; + } + SettlementAction::Submit => {} + } + + match chain.submit(&proof, calldata.clone()).await { + Ok(h) => break Some(h), + Err(e) => { + warn!( + "Settlement of block {} failed: {}; retrying", + proof.block_number.to_hex_string(), + e + ); + if cooldown(&finish_handle, poll_interval).await { + break 'outer; + } + continue; + } + } + }; + + let new_cursor = SettlementCursor { + block_number: u64::try_from(proof.block_number).unwrap_or_else(|_| { + panic!( + "Block number {} does not fit in u64", + proof.block_number.to_hex_string() + ) + }), + transaction_hash: tx_hash.unwrap_or(Felt::ZERO), + }; + + tokio::select! { + _ = finish_handle.shutdown_requested() => break, + _ = cursor_channel.send(new_cursor) => {}, + } + } + + debug!("TeePiltoverSettlementBackend graceful shutdown finished"); + finish_handle.finish(); +} + /// Settlement backend that submits TEE proofs to the Piltover contract via `update_state`. #[derive(Debug)] pub struct TeePiltoverSettlementBackend { @@ -254,205 +468,39 @@ impl TeeSettlementBackendBuilder for TeePiltoverSettlementBackendBuilder { } impl TeePiltoverSettlementBackend { - async fn get_piltover_block_number(&self) -> Result { - let raw = self - .provider - .call( - FunctionCall { - contract_address: self.piltover_address, - entry_point_selector: selector!("get_state"), - calldata: vec![], - }, - BlockId::Tag(BlockTag::Latest), - ) - .await?; - // AppchainState: [state_root, block_number, block_hash] — block_number is index 1. - raw.get(1) - .copied() - .ok_or_else(|| anyhow::anyhow!("get_state returned fewer than 2 felts")) - } - - async fn watch_tx(&self, tx_hash: Felt) -> Result<()> { - loop { - tokio::time::sleep(POLLING_INTERVAL).await; - match self.provider.get_transaction_receipt(tx_hash).await { - Ok(receipt) => match receipt.receipt { - TransactionReceipt::Invoke(r) => { - use starknet::core::types::ExecutionResult; - match r.execution_result { - ExecutionResult::Succeeded => return Ok(()), - ExecutionResult::Reverted { reason } => { - return Err(anyhow::anyhow!("Transaction reverted: {reason}")) - } - } - } - _ => return Ok(()), - }, - Err(starknet::providers::ProviderError::StarknetError( - starknet::core::types::StarknetError::TransactionHashNotFound, - )) => continue, - Err(e) => return Err(e.into()), - } - } - } - - /// Sleep one polling interval; returns `true` if shutdown was requested while - /// waiting, so retry loops can stop promptly instead of hanging. - async fn cooldown(&self) -> bool { - tokio::select! { - _ = self.finish_handle.shutdown_requested() => true, - _ = tokio::time::sleep(POLLING_INTERVAL) => false, - } - } - - async fn run(mut self) { - 'outer: loop { - let proof = tokio::select! { - _ = self.finish_handle.shutdown_requested() => break, - p = self.proof_channel.recv() => match p { - Some(p) => p, - None => { - debug!("Proof channel closed, shutting down"); - break; - } - }, - }; - - // Calldata is a pure function of the proof; a build failure means a - // malformed proof and retrying can't help, so skip it. The settlement - // errors below are the opposite — transient — so we RETRY the same proof - // and never drop it. Dropping a proof leaves a permanent gap: every later - // block's `prev_block_number` then mismatches the on-chain state and the - // contract rejects it forever ("State: invalid block number"). - let calldata = match build_tee_calldata(&proof, self.mock_prove) { - Ok(c) => c, - Err(e) => { - error!( - "Failed to build TEE calldata for block {}: {}", - proof.block_number.to_hex_string(), - e - ); - continue; - } - }; - - // Settle strictly in order. Only submit `update_state` once the Piltover - // contract's on-chain block equals this proof's parent; otherwise wait — - // a prior settlement is still landing, or `latest` lags `pre_confirmed` - // (fee estimation runs against `latest`, so submitting early reverts with - // "invalid block number"). Retry transient failures rather than advancing - // to the next proof. A `None` result means the block was already settled - // on-chain (e.g. resumed after a restart), so we just advance the cursor. - let tx_hash: Option = loop { - if self.finish_handle.is_shutdown_requested() { - break 'outer; - } - - let onchain = match self.get_piltover_block_number().await { - Ok(b) => b, - Err(e) => { - warn!( - "Failed to read Piltover block for {}: {}; retrying", - proof.block_number.to_hex_string(), - e - ); - if self.cooldown().await { - break 'outer; - } - continue; - } - }; - - match settlement_action(onchain, proof.prev_block_number, proof.block_number) { - SettlementAction::AlreadySettled => { - debug!( - "Block {} already settled on-chain; advancing cursor", - proof.block_number.to_hex_string() - ); - break None; - } - SettlementAction::WaitForParent => { - // Parent not yet on-chain — wait and recheck; do NOT skip ahead. - if self.cooldown().await { - break 'outer; - } - continue; - } - SettlementAction::Submit => {} - } - - let call = Call { - to: self.piltover_address, - selector: selector!("update_state"), - calldata: calldata.clone(), - }; - let execution = self.account.execute_v3(vec![call]); - - if let Err(e) = execution.estimate_fee().await { - warn!( - "Fee estimation failed for block {}: {}; retrying", - proof.block_number.to_hex_string(), - e - ); - if self.cooldown().await { - break 'outer; - } - continue; - } - let transaction = match execution.send().await { - Ok(t) => t, - Err(e) => { - warn!( - "Settlement transaction failed for block {}: {}; retrying", - proof.block_number.to_hex_string(), - e - ); - if self.cooldown().await { - break 'outer; - } - continue; - } - }; - match self.watch_tx(transaction.transaction_hash).await { - Ok(()) => break Some(transaction.transaction_hash), - Err(e) => { - warn!( - "Settlement tx confirmation failed for block {}: {}; retrying", - proof.block_number.to_hex_string(), - e - ); - if self.cooldown().await { - break 'outer; - } - continue; - } - } - }; - - let new_cursor = SettlementCursor { - block_number: u64::try_from(proof.block_number).unwrap_or_else(|_| { - panic!( - "Block number {} does not fit in u64", - proof.block_number.to_hex_string() - ) - }), - transaction_hash: tx_hash.unwrap_or(Felt::ZERO), - }; - - tokio::select! { - _ = self.finish_handle.shutdown_requested() => break, - _ = self.cursor_channel.send(new_cursor) => {}, - } - } - - debug!("TeePiltoverSettlementBackend graceful shutdown finished"); - self.finish_handle.finish(); + async fn run(self) { + // Move the chain-facing fields into a `PiltoverChain` so the settlement loop + // (`run_settlement`) is generic over the chain and can be unit-tested with a + // fake; the channels and the shutdown handle drive it. + let Self { + provider, + account, + piltover_address, + mock_prove, + proof_channel, + cursor_channel, + finish_handle, + } = self; + let chain = PiltoverChain { + provider, + account, + piltover_address, + }; + run_settlement( + &chain, + proof_channel, + cursor_channel, + finish_handle, + POLLING_INTERVAL, + mock_prove, + ) + .await; } } impl SettlementBackend for TeePiltoverSettlementBackend { async fn get_block_number(&self) -> Result { - self.get_piltover_block_number().await + piltover_block_number(&self.provider, self.piltover_address).await } } @@ -468,8 +516,16 @@ impl Daemon for TeePiltoverSettlementBackend { #[cfg(test)] mod tests { - use super::{settlement_action, SettlementAction}; + use super::{ + run_settlement, settlement_action, FinishHandle, SettlementAction, SettlementChain, + TeeProof, + }; + use anyhow::{anyhow, Result}; use starknet::core::types::Felt; + use std::collections::HashSet; + use std::sync::Mutex; + use std::time::Duration; + use tokio::sync::mpsc; fn f(n: u64) -> Felt { Felt::from(n) @@ -540,4 +596,123 @@ mod tests { SettlementAction::AlreadySettled ); } + + // --- integration tests: drive the `run_settlement` loop against a fake chain --- + + /// A dummy proof for `block` (parent = block-1, or the genesis sentinel for block 0). + /// `data` is empty (a valid mock-proof buffer) so `build_tee_calldata` succeeds; the + /// fake chain ignores the calldata and acts on the block numbers. + fn proof(block: u64) -> TeeProof { + TeeProof { + blocks: vec![], + data: vec![], + prev_state_root: Felt::ZERO, + state_root: Felt::ZERO, + prev_block_hash: Felt::ZERO, + block_hash: Felt::ZERO, + prev_block_number: if block == 0 { genesis() } else { f(block - 1) }, + block_number: f(block), + messages_commitment: Felt::ZERO, + l2_to_l1_messages: vec![], + l1_to_l2_messages: vec![], + katana_tee_config_hash: Felt::ZERO, + } + } + + /// A fake Piltover that simulates `update_state` ordering: it tracks an on-chain + /// block (starting at `at`) and advances exactly one block per accepted submit. + /// `failing_once` makes the first submit of a given block error transiently. + struct FakeChain { + onchain: Mutex, + settled: Mutex>, + fail_once: Mutex>, + } + + impl FakeChain { + fn at(start: Felt) -> Self { + Self { + onchain: Mutex::new(start), + settled: Mutex::new(Vec::new()), + fail_once: Mutex::new(HashSet::new()), + } + } + fn failing_once(self, blocks: &[u64]) -> Self { + *self.fail_once.lock().unwrap() = blocks.iter().copied().collect(); + self + } + fn settled(&self) -> Vec { + self.settled.lock().unwrap().clone() + } + } + + impl SettlementChain for FakeChain { + async fn onchain_block(&self) -> Result { + Ok(*self.onchain.lock().unwrap()) + } + async fn submit(&self, proof: &TeeProof, _calldata: Vec) -> Result { + let block = u64::try_from(proof.block_number).unwrap(); + if self.fail_once.lock().unwrap().remove(&block) { + return Err(anyhow!("transient submit failure for block {block}")); + } + // The loop only submits when the chain is at the proof's parent, so advance. + *self.onchain.lock().unwrap() = proof.block_number; + self.settled.lock().unwrap().push(block); + Ok(f(block)) + } + } + + /// Run `run_settlement` over `proofs` against `chain` to completion; returns the + /// cursor block numbers it emitted. A zero poll interval keeps retries instant. + async fn drive(chain: &FakeChain, proofs: Vec) -> Vec { + let (proof_tx, proof_rx) = mpsc::channel(64); + let (cursor_tx, mut cursor_rx) = mpsc::channel(64); + for p in proofs { + proof_tx.send(p).await.unwrap(); + } + drop(proof_tx); // close the proof channel so the loop finishes + run_settlement( + chain, + proof_rx, + cursor_tx, + FinishHandle::new(), + Duration::from_millis(0), + true, // mock_prove + ) + .await; + let mut cursors = Vec::new(); + while let Ok(c) = cursor_rx.try_recv() { + cursors.push(c.block_number); + } + cursors + } + + #[tokio::test] + async fn settles_a_fresh_chain_from_genesis_in_order() { + // A fresh Piltover sits at the -1 sentinel; the loop must settle block 0 and + // every block after it, in order. + let chain = FakeChain::at(genesis()); + let cursors = drive(&chain, (0..5).map(proof).collect()).await; + assert_eq!(chain.settled(), vec![0, 1, 2, 3, 4]); + assert_eq!(cursors, vec![0, 1, 2, 3, 4]); + } + + #[tokio::test] + async fn retries_a_transient_submit_failure_without_dropping_the_block() { + // Block 2's first submit fails. The loop must retry the SAME block — not drop it + // and skip ahead (which would wedge every later block) — so all blocks settle. + let chain = FakeChain::at(genesis()).failing_once(&[2]); + let cursors = drive(&chain, (0..5).map(proof).collect()).await; + assert_eq!(chain.settled(), vec![0, 1, 2, 3, 4]); + assert_eq!(cursors, vec![0, 1, 2, 3, 4]); + } + + #[tokio::test] + async fn skips_blocks_already_settled_on_chain_after_a_restart() { + // Resumed with the chain already at block 3: proofs for 2 and 3 are skipped + // (not re-submitted), and 4, 5 are settled. Every proof still advances a cursor. + let chain = FakeChain::at(f(3)); + let cursors = drive(&chain, (2..6).map(proof).collect()).await; + assert_eq!(chain.settled(), vec![4, 5]); + assert_eq!(cursors, vec![2, 3, 4, 5]); + } }