From b16be5aaa38dfaa96343f5ceb13bd919062b3399 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 15 Jul 2026 00:34:33 +0300 Subject: [PATCH 001/148] chore: disable nightly Docker cron (#26400) --- .github/workflows/docker.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 06d501f6415..3c6ab4c5c82 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -2,7 +2,6 @@ # # Triggers: # - Push tag v*: builds release (RC or latest) -# - Schedule: builds nightly + profiling # - Manual: builds git-sha or nightly name: docker @@ -11,8 +10,6 @@ on: push: tags: - v* - schedule: - - cron: "0 1 * * *" workflow_dispatch: inputs: build_type: From 928ff1a65fa6e945d67a73e5f7d1d49d37e7d240 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 15 Jul 2026 02:40:40 +0300 Subject: [PATCH 002/148] chore(txpool): remove unused pool test utilities (#26396) --- crates/transaction-pool/src/test_utils/mod.rs | 2 - .../transaction-pool/src/test_utils/pool.rs | 604 ------------------ 2 files changed, 606 deletions(-) delete mode 100644 crates/transaction-pool/src/test_utils/pool.rs diff --git a/crates/transaction-pool/src/test_utils/mod.rs b/crates/transaction-pool/src/test_utils/mod.rs index 7c0ce6ae762..6b33aa01e6f 100644 --- a/crates/transaction-pool/src/test_utils/mod.rs +++ b/crates/transaction-pool/src/test_utils/mod.rs @@ -9,8 +9,6 @@ pub use tx_gen::*; mod mock; pub use mock::*; -mod pool; - mod okvalidator; pub use okvalidator::*; diff --git a/crates/transaction-pool/src/test_utils/pool.rs b/crates/transaction-pool/src/test_utils/pool.rs deleted file mode 100644 index 3b345d5bcd1..00000000000 --- a/crates/transaction-pool/src/test_utils/pool.rs +++ /dev/null @@ -1,604 +0,0 @@ -//! Test helpers for mocking an entire pool. - -#![allow(dead_code)] - -use crate::{ - error::PoolErrorKind, - pool::{state::SubPool, txpool::TxPool, AddedTransaction}, - test_utils::{MockOrdering, MockTransactionDistribution, MockTransactionFactory}, - TransactionOrdering, -}; -use alloy_primitives::{map::AddressMap, Address, U256}; -use rand::Rng; -use std::ops::{Deref, DerefMut}; - -/// A wrapped `TxPool` with additional helpers for testing -pub(crate) struct MockPool { - // The wrapped pool. - pool: TxPool, -} - -impl MockPool { - /// The total size of all subpools - fn total_subpool_size(&self) -> usize { - self.pool.pending().len() + self.pool.base_fee().len() + self.pool.queued().len() - } - - /// Checks that all pool invariants hold. - fn enforce_invariants(&self) { - assert_eq!( - self.pool.len(), - self.total_subpool_size(), - "Tx in AllTransactions and sum(subpools) must match" - ); - } -} - -impl Default for MockPool { - fn default() -> Self { - Self { pool: TxPool::new(MockOrdering::default(), Default::default()) } - } -} - -impl Deref for MockPool { - type Target = TxPool; - - fn deref(&self) -> &Self::Target { - &self.pool - } -} - -impl DerefMut for MockPool { - fn deref_mut(&mut self) -> &mut Self::Target { - &mut self.pool - } -} - -/// Simulates transaction execution. -pub(crate) struct MockTransactionSimulator { - /// The pending base fee - base_fee: u128, - /// Generator for transactions - tx_generator: MockTransactionDistribution, - /// represents the on chain balance of a sender. - balances: AddressMap, - /// represents the on chain nonce of a sender. - nonces: AddressMap, - /// A set of addresses to use as senders. - senders: Vec
, - /// What scenarios to execute. - scenarios: Vec, - /// All previous scenarios executed by a sender. - executed: AddressMap, - /// "Validates" generated transactions. - validator: MockTransactionFactory, - /// Represents the gaps in nonces for each sender. - nonce_gaps: AddressMap, - /// The rng instance used to select senders and scenarios. - rng: R, -} - -impl MockTransactionSimulator { - /// Returns a new mock instance - pub(crate) fn new(mut rng: R, config: MockSimulatorConfig) -> Self { - let senders = config.addresses(&mut rng); - Self { - base_fee: config.base_fee, - balances: senders.iter().copied().map(|a| (a, rng.random())).collect(), - nonces: senders.iter().copied().map(|a| (a, 0)).collect(), - senders, - scenarios: config.scenarios, - tx_generator: config.tx_generator, - executed: Default::default(), - validator: Default::default(), - nonce_gaps: Default::default(), - rng, - } - } - - /// Creates a pool configured for this simulator - /// - /// This is needed because `MockPool::default()` sets `pending_basefee` to 7, but we might want - /// to use different values - pub(crate) fn create_pool(&self) -> MockPool { - let mut pool = MockPool::default(); - let mut info = pool.block_info(); - info.pending_basefee = self.base_fee as u64; - pool.set_block_info(info); - pool - } - - /// Returns a random address from the senders set - fn rng_address(&mut self) -> Address { - let idx = self.rng.random_range(0..self.senders.len()); - self.senders[idx] - } - - /// Returns a random scenario from the scenario set - fn rng_scenario(&mut self) -> ScenarioType { - let idx = self.rng.random_range(0..self.scenarios.len()); - self.scenarios[idx].clone() - } - - /// Executes the next scenario and applies it to the pool - pub(crate) fn next(&mut self, pool: &mut MockPool) { - let sender = self.rng_address(); - let scenario = self.rng_scenario(); - let on_chain_nonce = self.nonces[&sender]; - let on_chain_balance = self.balances[&sender]; - - match scenario { - ScenarioType::OnchainNonce => { - // uses fee from fee_ranges - let tx = self.tx_generator.tx(on_chain_nonce, &mut self.rng).with_sender(sender); - let valid_tx = self.validator.validated(tx); - - let res = - match pool.add_transaction(valid_tx, on_chain_balance, on_chain_nonce, None) { - Ok(res) => res, - Err(e) => match e.kind { - // skip pool capacity/replacement errors (not relevant) - PoolErrorKind::SpammerExceededCapacity(_) | - PoolErrorKind::ReplacementUnderpriced => return, - _ => panic!("unexpected error: {e:?}"), - }, - }; - - match res { - AddedTransaction::Pending(_) => {} - AddedTransaction::Parked { .. } => { - panic!("expected pending") - } - } - - self.executed - .entry(sender) - .or_insert_with(|| ExecutedScenarios { sender, scenarios: vec![] }) // in the case of a new sender - .scenarios - .push(ExecutedScenario { - balance: on_chain_balance, - nonce: on_chain_nonce, - scenario: Scenario::OnchainNonce { nonce: on_chain_nonce }, - }); - - self.nonces.insert(sender, on_chain_nonce + 1); - } - - ScenarioType::HigherNonce { skip } => { - // if this sender already has a nonce gap, skip - if self.nonce_gaps.contains_key(&sender) { - return; - } - - let higher_nonce = on_chain_nonce + skip; - - // uses fee from fee_ranges - let tx = self.tx_generator.tx(higher_nonce, &mut self.rng).with_sender(sender); - let valid_tx = self.validator.validated(tx); - - let res = - match pool.add_transaction(valid_tx, on_chain_balance, on_chain_nonce, None) { - Ok(res) => res, - Err(e) => match e.kind { - // skip pool capacity/replacement errors (not relevant) - PoolErrorKind::SpammerExceededCapacity(_) | - PoolErrorKind::ReplacementUnderpriced => return, - _ => panic!("unexpected error: {e:?}"), - }, - }; - - match res { - AddedTransaction::Pending(_) => { - panic!("expected parked") - } - AddedTransaction::Parked { subpool, .. } => { - assert_eq!( - subpool, - SubPool::Queued, - "expected to be moved to queued subpool" - ); - } - } - - self.executed - .entry(sender) - .or_insert_with(|| ExecutedScenarios { sender, scenarios: vec![] }) // in the case of a new sender - .scenarios - .push(ExecutedScenario { - balance: on_chain_balance, - nonce: on_chain_nonce, - scenario: Scenario::HigherNonce { - onchain: on_chain_nonce, - nonce: higher_nonce, - }, - }); - self.nonce_gaps.insert(sender, higher_nonce); - } - - ScenarioType::BelowBaseFee { fee } => { - // fee should be in [MIN_PROTOCOL_BASE_FEE, base_fee) - let tx = self - .tx_generator - .tx(on_chain_nonce, &mut self.rng) - .with_sender(sender) - .with_gas_price(fee); - let valid_tx = self.validator.validated(tx); - - let res = - match pool.add_transaction(valid_tx, on_chain_balance, on_chain_nonce, None) { - Ok(res) => res, - Err(e) => match e.kind { - // skip pool capacity/replacement errors (not relevant) - PoolErrorKind::SpammerExceededCapacity(_) | - PoolErrorKind::ReplacementUnderpriced => return, - _ => panic!("unexpected error: {e:?}"), - }, - }; - - match res { - AddedTransaction::Pending(_) => panic!("expected parked"), - AddedTransaction::Parked { subpool, .. } => { - assert_eq!( - subpool, - SubPool::BaseFee, - "expected to be moved to base fee subpool" - ); - } - } - self.executed - .entry(sender) - .or_insert_with(|| ExecutedScenarios { sender, scenarios: vec![] }) // in the case of a new sender - .scenarios - .push(ExecutedScenario { - balance: on_chain_balance, - nonce: on_chain_nonce, - scenario: Scenario::BelowBaseFee { fee }, - }); - } - - ScenarioType::FillNonceGap => { - if self.nonce_gaps.is_empty() { - return; - } - - let gap_senders: Vec
= self.nonce_gaps.keys().copied().collect(); - let idx = self.rng.random_range(0..gap_senders.len()); - let gap_sender = gap_senders[idx]; - let queued_nonce = self.nonce_gaps[&gap_sender]; - - let sender_onchain_nonce = self.nonces[&gap_sender]; - let sender_balance = self.balances[&gap_sender]; - - for fill_nonce in sender_onchain_nonce..queued_nonce { - let tx = - self.tx_generator.tx(fill_nonce, &mut self.rng).with_sender(gap_sender); - let valid_tx = self.validator.validated(tx); - - let res = match pool.add_transaction( - valid_tx, - sender_balance, - sender_onchain_nonce, - None, - ) { - Ok(res) => res, - Err(e) => match e.kind { - // skip pool capacity/replacement errors (not relevant) - PoolErrorKind::SpammerExceededCapacity(_) | - PoolErrorKind::ReplacementUnderpriced => return, - _ => panic!("unexpected error: {e:?}"), - }, - }; - - match res { - AddedTransaction::Pending(_) => {} - AddedTransaction::Parked { .. } => { - panic!("expected pending when filling gap") - } - } - - self.executed - .entry(gap_sender) - .or_insert_with(|| ExecutedScenarios { - sender: gap_sender, - scenarios: vec![], - }) - .scenarios - .push(ExecutedScenario { - balance: sender_balance, - nonce: fill_nonce, - scenario: Scenario::FillNonceGap { - filled_nonce: fill_nonce, - promoted_nonce: queued_nonce, - }, - }); - } - self.nonces.insert(gap_sender, queued_nonce + 1); - self.nonce_gaps.remove(&gap_sender); - } - } - // make sure everything is set - pool.enforce_invariants(); - } -} - -/// How to configure a new mock transaction stream -pub(crate) struct MockSimulatorConfig { - /// How many senders to generate. - pub(crate) num_senders: usize, - /// Scenarios to test - pub(crate) scenarios: Vec, - /// The start base fee - pub(crate) base_fee: u128, - /// generator for transactions - pub(crate) tx_generator: MockTransactionDistribution, -} - -impl MockSimulatorConfig { - /// Generates a set of random addresses - pub(crate) fn addresses(&self, rng: &mut impl rand::Rng) -> Vec
{ - std::iter::repeat_with(|| Address::random_with(rng)).take(self.num_senders).collect() - } -} - -/// Represents the different types of test scenarios. -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) enum ScenarioType { - OnchainNonce, - HigherNonce { skip: u64 }, - BelowBaseFee { fee: u128 }, - FillNonceGap, -} - -/// The actual scenario, ready to be executed -/// -/// A scenario produces one or more transactions and expects a certain Outcome. -/// -/// An executed scenario can affect previous executed transactions -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) enum Scenario { - /// Send a tx with the same nonce as on chain. - OnchainNonce { nonce: u64 }, - /// Send a tx with a higher nonce that what the sender has on chain - HigherNonce { onchain: u64, nonce: u64 }, - /// Send a tx with a base fee below the base fee of the pool - BelowBaseFee { fee: u128 }, - /// Fill a nonce gap to promote queued transactions - FillNonceGap { filled_nonce: u64, promoted_nonce: u64 }, - /// Execute multiple test scenarios - Multi { scenario: Vec }, -} - -/// Represents an executed scenario -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) struct ExecutedScenario { - /// balance at the time of execution - balance: U256, - /// nonce at the time of execution - nonce: u64, - /// The executed scenario - scenario: Scenario, -} - -/// All executed scenarios by a sender -#[derive(Debug, Clone)] -#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] -pub(crate) struct ExecutedScenarios { - sender: Address, - scenarios: Vec, -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::test_utils::{MockFeeRange, MockTransactionRatio}; - - #[test] - fn test_on_chain_nonce_scenario() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 10, - scenarios: vec![ScenarioType::OnchainNonce], - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - simulator.next(&mut pool); - assert_eq!(pool.pending().len(), 1); - assert_eq!(pool.queued().len(), 0); - assert_eq!(pool.base_fee().len(), 0); - } - - #[test] - fn test_higher_nonce_scenario() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 10, - scenarios: vec![ScenarioType::HigherNonce { skip: 1 }], - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - simulator.next(&mut pool); - assert_eq!(pool.pending().len(), 0); - assert_eq!(pool.queued().len(), 1); - assert_eq!(pool.base_fee().len(), 0); - } - - #[test] - fn test_below_base_fee_scenario() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 10, - scenarios: vec![ScenarioType::BelowBaseFee { fee: 8 }], /* fee should be in - * [MIN_PROTOCOL_BASE_FEE, - * base_fee) */ - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - simulator.next(&mut pool); - assert_eq!(pool.pending().len(), 0); - assert_eq!(pool.queued().len(), 0); - assert_eq!(pool.base_fee().len(), 1); - } - - #[test] - fn test_fill_nonce_gap_scenario() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 5, - scenarios: vec![ScenarioType::HigherNonce { skip: 5 }], - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - // create some nonce gaps - for _ in 0..10 { - simulator.next(&mut pool); - } - - let num_gaps = simulator.nonce_gaps.len(); - - assert_eq!(pool.pending().len(), 0); - assert_eq!(pool.queued().len(), num_gaps); - assert_eq!(pool.base_fee().len(), 0); - - simulator.scenarios = vec![ScenarioType::FillNonceGap]; - for _ in 0..num_gaps { - simulator.next(&mut pool); - } - - let expected_pending = num_gaps * 6; - assert_eq!(pool.pending().len(), expected_pending); - assert_eq!(pool.queued().len(), 0); - assert_eq!(pool.base_fee().len(), 0); - } - - #[test] - fn test_random_scenarios() { - let transaction_ratio = MockTransactionRatio { - legacy_pct: 30, - dynamic_fee_pct: 70, - access_list_pct: 0, - blob_pct: 0, - }; - - let base_fee = 10u128; - let fee_ranges = MockFeeRange { - gas_price: (base_fee..100).try_into().unwrap(), - priority_fee: (1u128..10).try_into().unwrap(), - max_fee: (base_fee..110).try_into().unwrap(), - max_fee_blob: (1u128..100).try_into().unwrap(), - }; - - let config = MockSimulatorConfig { - num_senders: 10, - scenarios: vec![ - ScenarioType::OnchainNonce, - ScenarioType::HigherNonce { skip: 2 }, - ScenarioType::BelowBaseFee { fee: 8 }, - ScenarioType::FillNonceGap, - ], - base_fee, - tx_generator: MockTransactionDistribution::new( - transaction_ratio, - fee_ranges, - 10..100, - 10..100, - ), - }; - let mut simulator = MockTransactionSimulator::new(rand::rng(), config); - let mut pool = simulator.create_pool(); - - for _ in 0..1000 { - simulator.next(&mut pool); - } - } -} From a47885fd05e4ce17fd29c1092ab66e93f939983a Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 15 Jul 2026 12:47:02 +0200 Subject: [PATCH 003/148] refactor(trie): remove changed path tracking (#26403) --- crates/chain-state/src/test_utils.rs | 1 - .../engine/tree/src/tree/payload_validator.rs | 18 +-- .../tree/src/tree/state_root_strategy/mod.rs | 18 +-- .../tree/state_root_strategy/sparse_trie.rs | 2 - crates/payload/primitives/src/traits.rs | 4 +- .../src/providers/database/provider.rs | 1 - crates/trie/common/src/trie_data.rs | 53 +------ crates/trie/parallel/src/state_root_task.rs | 6 +- crates/trie/sparse/src/arena/mod.rs | 130 ++---------------- crates/trie/sparse/src/arena/nodes.rs | 12 -- crates/trie/sparse/src/state.rs | 104 ++------------ crates/trie/sparse/src/traits.rs | 14 +- crates/trie/sparse/src/trie.rs | 35 +---- .../trie/sparse/tests/suite/changed_paths.rs | 127 ----------------- crates/trie/sparse/tests/suite/main.rs | 7 - 15 files changed, 39 insertions(+), 493 deletions(-) delete mode 100644 crates/trie/sparse/tests/suite/changed_paths.rs diff --git a/crates/chain-state/src/test_utils.rs b/crates/chain-state/src/test_utils.rs index 577bcefad5e..18fb6b5d43f 100644 --- a/crates/chain-state/src/test_utils.rs +++ b/crates/chain-state/src/test_utils.rs @@ -314,7 +314,6 @@ impl TestBlockBuilder { let trie_data = ComputedTrieData { sorted: SortedTrieData { hashed_state: Arc::new(hashed_state), ..Default::default() }, - ..Default::default() }; let block_hash = recovered.hash(); diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 9711d2882d4..f8efd2cc093 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -154,8 +154,8 @@ use reth_provider::{ }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_trie::{ - hashed_cursor::HashedCursorFactory, prefix_set::TriePrefixSetsMut, - trie_cursor::TrieCursorFactory, updates::TrieUpdates, LazyTrieData, + hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, + LazyTrieData, }; use reth_trie_db::ChangesetCache; use std::{ @@ -809,7 +809,6 @@ where let state_root = root_outcome.state_root; let trie_output = root_outcome.trie_updates; - let changed_paths = root_outcome.changed_paths; // A fallback path recomputed the hashed post state. Replace the streaming-derived one // and re-run hashed-state validation against it, since a failed state-root task may @@ -877,13 +876,8 @@ where let _ = valid_block_tx.send(()); } - let executed_block = self.spawn_deferred_trie_task( - Arc::new(block), - output, - hashed_state, - trie_output, - changed_paths, - ); + let executed_block = + self.spawn_deferred_trie_task(Arc::new(block), output, hashed_state, trie_output); let raw_bal = decoded_bal.map(|decoded_bal| decoded_bal.as_raw_bal().clone()); Ok(ValidationOutput::new(executed_block, timing_stats).with_raw_bal(raw_bal)) } @@ -1453,7 +1447,6 @@ where execution_outcome: Arc>, hashed_state: LazyHashedPostState, trie_output: Arc, - changed_paths: Option>, ) -> ExecutedBlock { // Create deferred handle and task that owns the unsorted inputs. // Resolve the lazy handle into Arc. By this point the hashed state has @@ -1463,7 +1456,7 @@ where Err(handle) => handle.get().clone(), }; let (deferred_trie_data, deferred_trie_task) = - LazyTrieData::pending(hashed_state, trie_output, changed_paths); + LazyTrieData::pending(hashed_state, trie_output); let block_validation_metrics = self.metrics.block_validation.clone(); // Capture block info for tracing. @@ -1788,7 +1781,6 @@ where block.execution_output, LazyHashedPostState::ready(block.hashed_state), block.trie_updates, - block.changed_paths, )) } diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 7909811516e..3893bb5214b 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -443,8 +443,6 @@ pub struct StateRootJobOutcome { pub state_root: B256, /// Trie updates associated with the computed state root. pub trie_updates: Arc, - /// Changed trie node base paths retained while computing the root, if the job tracks them. - pub changed_paths: Option>, /// Hashed post state recomputed by a fallback path. /// /// When set, the root was not derived from the streamed updates, so validation replaces its @@ -453,15 +451,9 @@ pub struct StateRootJobOutcome { } impl StateRootJobOutcome { - /// Creates a state-root job outcome without changed paths. + /// Creates a state-root job outcome. pub const fn new(state_root: B256, trie_updates: Arc) -> Self { - Self { state_root, trie_updates, changed_paths: None, hashed_state: None } - } - - /// Sets the changed trie node base paths retained while computing the root. - pub fn with_changed_paths(mut self, changed_paths: Option>) -> Self { - self.changed_paths = changed_paths; - self + Self { state_root, trie_updates, hashed_state: None } } /// Sets the hashed post state recomputed by a fallback path. @@ -623,7 +615,6 @@ impl DefaultStateRootStrategy { .with_default_storage_trie(default_trie) .with_updates(true) }); - sparse_state_trie.set_changed_paths(true); sparse_state_trie.set_hot_cache_capacities(max_hot_slots, max_hot_accounts); let mut task = SparseTrieCacheTask::new_with_trie( @@ -980,7 +971,6 @@ where state_root, trie_updates, hashed_state: _hashed_state, - changed_paths, #[cfg(feature = "trie-debug")] debug_recorders, } = outcome; @@ -1003,7 +993,7 @@ where write_trie_debug_recorders(_block.header().number(), &debug_recorders); } - StateRootJobOutcome::new(state_root, trie_updates).with_changed_paths(changed_paths) + StateRootJobOutcome::new(state_root, trie_updates) } } @@ -1259,7 +1249,7 @@ mod tests { } #[test] - fn sparse_trie_retained_paths_uses_hashed_state_when_changed_paths_missing() { + fn sparse_trie_retained_paths_uses_hashed_state() { let blocks: Vec<_> = TestBlockBuilder::eth() .get_executed_blocks(1..2) .map(|block| with_hashed_state(block, trie_hashed_state(0x01, 0x02))) diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 9bb614e047d..03f6f839619 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -354,7 +354,6 @@ where #[cfg(feature = "trie-debug")] let debug_recorders = self.trie.take_debug_recorders(); - let changed_paths = Some(Arc::new(self.trie.take_changed_paths().unwrap_or_default())); let end = Instant::now(); self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start)); @@ -374,7 +373,6 @@ where trie_updates: Arc::new(trie_updates), hashed_state: finalized_hashed_state .expect("finished state updates publish the hashed post state"), - changed_paths, #[cfg(feature = "trie-debug")] debug_recorders, }) diff --git a/crates/payload/primitives/src/traits.rs b/crates/payload/primitives/src/traits.rs index 0ac1bc80103..4d46e6eb0fa 100644 --- a/crates/payload/primitives/src/traits.rs +++ b/crates/payload/primitives/src/traits.rs @@ -10,7 +10,7 @@ use core::fmt; use either::Either; use reth_execution_types::BlockExecutionOutput; use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader}; -use reth_trie_common::{prefix_set::TriePrefixSetsMut, updates::TrieUpdates, HashedPostState}; +use reth_trie_common::{updates::TrieUpdates, HashedPostState}; /// Represents an executed block for payload building purposes. /// @@ -26,8 +26,6 @@ pub struct BuiltPayloadExecutedBlock { pub hashed_state: Arc, /// Trie updates that result from calculating the state root for the block (unsorted). pub trie_updates: Arc, - /// Changed trie node base paths, if known. - pub changed_paths: Option>, } /// Represents a successfully built execution payload (block). diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index a9a8b2ebbe5..a241ed7cff3 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -5125,7 +5125,6 @@ mod tests { }), ComputedTrieData { sorted: SortedTrieData::new(Arc::new(hashed_state), Default::default()), - ..Default::default() }, ); blocks.push(executed); diff --git a/crates/trie/common/src/trie_data.rs b/crates/trie/common/src/trie_data.rs index 865cd2a0d9c..62ca96be60d 100644 --- a/crates/trie/common/src/trie_data.rs +++ b/crates/trie/common/src/trie_data.rs @@ -4,7 +4,6 @@ //! trie-related data containing sorted hashed state and trie updates. use crate::{ - prefix_set::TriePrefixSetsMut, updates::{TrieUpdates, TrieUpdatesSorted}, HashedPostState, HashedPostStateSorted, }; @@ -35,13 +34,11 @@ impl SortedTrieData { } } -/// Container for sorted trie data that also includes `changed_paths`. +/// Container for sorted trie data. #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct ComputedTrieData { /// Sorted trie data: hashed state and trie updates. pub sorted: SortedTrieData, - /// Changed trie node base paths produced by state root computation. - pub changed_paths: Option>, } impl ComputedTrieData { @@ -50,16 +47,7 @@ impl ComputedTrieData { hashed_state: Arc, trie_updates: Arc, ) -> Self { - Self::new_with_changed_paths(hashed_state, trie_updates, None) - } - - /// Construct sorted trie data with changed trie node base paths for one block. - pub const fn new_with_changed_paths( - hashed_state: Arc, - trie_updates: Arc, - changed_paths: Option>, - ) -> Self { - Self { sorted: SortedTrieData::new(hashed_state, trie_updates), changed_paths } + Self { sorted: SortedTrieData::new(hashed_state, trie_updates) } } } @@ -128,15 +116,11 @@ impl LazyTrieData { pub fn pending( hashed_state: Arc, trie_updates: Arc, - changed_paths: Option>, ) -> (Self, LazyTrieDataProducer) { let value = Arc::new(OnceLock::new()); ( Self { data: Arc::clone(&value), mode: LazyTrieDataMode::Pending }, - LazyTrieDataProducer { - value, - inputs: PendingInputs { hashed_state, trie_updates, changed_paths }, - }, + LazyTrieDataProducer { value, inputs: PendingInputs { hashed_state, trie_updates } }, ) } @@ -235,7 +219,7 @@ impl LazyTrieDataProducer { /// Computes sorted trie data, publishes it to waiters, and returns it to the task owner. pub fn compute_and_publish(self) -> ComputedTrieData { let Self { value, inputs } = self; - let computed = Self::sort(inputs.hashed_state, inputs.trie_updates, inputs.changed_paths); + let computed = Self::sort(inputs.hashed_state, inputs.trie_updates); let _ = value.set(computed.clone()); computed } @@ -244,7 +228,6 @@ impl LazyTrieDataProducer { pub fn sort( hashed_state: Arc, trie_updates: Arc, - changed_paths: Option>, ) -> ComputedTrieData { #[cfg(feature = "rayon")] let (sorted_hashed_state, sorted_trie_updates) = rayon::join( @@ -270,11 +253,7 @@ impl LazyTrieDataProducer { }, ); - ComputedTrieData::new_with_changed_paths( - Arc::new(sorted_hashed_state), - Arc::new(sorted_trie_updates), - changed_paths, - ) + ComputedTrieData::new(Arc::new(sorted_hashed_state), Arc::new(sorted_trie_updates)) } } @@ -285,8 +264,6 @@ struct PendingInputs { hashed_state: Arc, /// Unsorted trie updates from state root computation. trie_updates: Arc, - /// Changed trie node base paths from state root computation. - changed_paths: Option>, } #[cfg(test)] @@ -306,21 +283,9 @@ mod tests { LazyTrieData::pending( Arc::new(HashedPostState::default()), Arc::new(TrieUpdates::default()), - None, ) } - fn assert_changed_paths_ptr_eq( - left: &Option>, - right: &Option>, - ) { - match (left, right) { - (Some(left), Some(right)) => assert!(Arc::ptr_eq(left, right)), - (None, None) => {} - _ => panic!("changed paths presence mismatch"), - } - } - #[test] fn test_lazy_ready_is_initialized() { let lazy = LazyTrieData::ready(ComputedTrieData::default()); @@ -366,10 +331,8 @@ mod tests { assert!(Arc::ptr_eq(&published.sorted.hashed_state, &first.sorted.hashed_state)); assert!(Arc::ptr_eq(&published.sorted.trie_updates, &first.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&published.changed_paths, &first.changed_paths); assert!(Arc::ptr_eq(&first.sorted.hashed_state, &second.sorted.hashed_state)); assert!(Arc::ptr_eq(&first.sorted.trie_updates, &second.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&first.changed_paths, &second.changed_paths); } #[test] @@ -385,7 +348,6 @@ mod tests { assert!(Arc::ptr_eq(&published.sorted.hashed_state, &result.sorted.hashed_state)); assert!(Arc::ptr_eq(&published.sorted.trie_updates, &result.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&published.changed_paths, &result.changed_paths); } #[test] @@ -400,10 +362,8 @@ mod tests { assert!(Arc::ptr_eq(&published.sorted.hashed_state, &result1.sorted.hashed_state)); assert!(Arc::ptr_eq(&published.sorted.trie_updates, &result1.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&published.changed_paths, &result1.changed_paths); assert!(Arc::ptr_eq(&result1.sorted.hashed_state, &result2.sorted.hashed_state)); assert!(Arc::ptr_eq(&result1.sorted.trie_updates, &result2.sorted.trie_updates)); - assert_changed_paths_ptr_eq(&result1.changed_paths, &result2.changed_paths); } #[test] @@ -418,7 +378,7 @@ mod tests { )]); let (deferred, task) = - LazyTrieData::pending(Arc::new(hashed_state), Arc::new(TrieUpdates::default()), None); + LazyTrieData::pending(Arc::new(hashed_state), Arc::new(TrieUpdates::default())); let _ = task.compute_and_publish(); let result = deferred.get().clone(); @@ -435,7 +395,6 @@ mod tests { let (deferred, task) = LazyTrieData::pending( Arc::new(HashedPostState { accounts, storages: Default::default() }), Arc::new(TrieUpdates::default()), - None, ); let _ = task.compute_and_publish(); diff --git a/crates/trie/parallel/src/state_root_task.rs b/crates/trie/parallel/src/state_root_task.rs index 0fcf0e61f85..3bd7b2e99e2 100644 --- a/crates/trie/parallel/src/state_root_task.rs +++ b/crates/trie/parallel/src/state_root_task.rs @@ -11,8 +11,7 @@ use crate::error::StateRootTaskError; use alloy_evm::block::OnStateHook; use alloy_primitives::{keccak256, map::B256Map, B256}; use reth_trie::{ - prefix_set::TriePrefixSetsMut, updates::TrieUpdates, HashedPostState, HashedStorage, - MultiProofTargetsV2, ProofV2Target, + updates::TrieUpdates, HashedPostState, HashedStorage, MultiProofTargetsV2, ProofV2Target, }; use revm::state::EvmState; use std::{fmt, sync::Arc}; @@ -44,8 +43,6 @@ pub struct StateRootComputeOutcome { pub trie_updates: Arc, /// Hashed post state produced while computing the state root. pub hashed_state: Arc, - /// Changed trie node base paths retained while computing the root. - pub changed_paths: Option>, /// Debug recorders taken from the sparse tries, keyed by `None` for account trie /// and `Some(address)` for storage tries. #[cfg(feature = "trie-debug")] @@ -674,7 +671,6 @@ mod tests { state_root: B256::repeat_byte(0x42), trie_updates: Arc::new(TrieUpdates::default()), hashed_state: Arc::new(HashedPostState::default()), - changed_paths: None, #[cfg(feature = "trie-debug")] debug_recorders: Vec::new(), })) diff --git a/crates/trie/sparse/src/arena/mod.rs b/crates/trie/sparse/src/arena/mod.rs index c616fdb4e6c..cd4bacb4d2e 100644 --- a/crates/trie/sparse/src/arena/mod.rs +++ b/crates/trie/sparse/src/arena/mod.rs @@ -15,8 +15,8 @@ use alloy_trie::TrieMask; use core::{cmp::Reverse, mem}; use reth_execution_errors::SparseTrieResult; use reth_trie_common::{ - prefix_set::PrefixSetMut, BranchNodeMasks, BranchNodeRef, ExtensionNodeRef, LeafNodeRef, - Nibbles, ProofTrieNodeV2, RlpNode, TrieNodeV2, EMPTY_ROOT_HASH, + BranchNodeMasks, BranchNodeRef, ExtensionNodeRef, LeafNodeRef, Nibbles, ProofTrieNodeV2, + RlpNode, TrieNodeV2, EMPTY_ROOT_HASH, }; use slotmap::{DefaultKey, SlotMap}; use smallvec::SmallVec; @@ -125,8 +125,6 @@ struct ArenaTrieBuffers { /// Trie updates built up directly during hashing and structural changes. `Some` when /// tracking updates, `None` otherwise. Initialized alongside `updates` in `set_updates`. updates: Option, - /// Changed node base paths accumulated during hashing. - changed_paths: Option, /// Reusable buffer for RLP encoding. rlp_buf: Vec, /// Reusable buffer for child `RlpNode`s during hashing. @@ -138,9 +136,6 @@ impl ArenaTrieBuffers { if let Some(updates) = self.updates.as_mut() { updates.clear(); } - if let Some(changed_paths) = self.changed_paths.as_mut() { - changed_paths.clear(); - } self.rlp_buf.clear(); self.rlp_node_buf.clear(); } @@ -176,12 +171,11 @@ impl ArenaSparseSubtrie { /// Creates a new subtrie with a pre-allocated root slot containing /// [`ArenaSparseNode::EmptyRoot`]. The caller must overwrite `subtrie.arena[subtrie.root]` /// before use. - fn new(record_updates: bool, record_changed_paths: bool) -> Box { + fn new(record_updates: bool) -> Box { let mut arena = SlotMap::new(); let root = arena.insert(ArenaSparseNode::EmptyRoot); let buffers = ArenaTrieBuffers { updates: record_updates.then(SparseTrieUpdates::default), - changed_paths: record_changed_paths.then(PrefixSetMut::default), ..Default::default() }; Box::new(Self { @@ -695,43 +689,6 @@ impl ArenaParallelSparseTrie { self } - /// Set whether changed node base paths should be retained during hashing. - pub fn set_changed_paths(&mut self, retain_changed_paths: bool) { - if retain_changed_paths { - self.buffers.changed_paths.get_or_insert_with(PrefixSetMut::default).clear(); - } else { - self.buffers.changed_paths = None; - } - - for (_, node) in &mut self.upper_arena { - let ArenaSparseNode::Subtrie(subtrie) = node else { - continue; - }; - if retain_changed_paths { - subtrie.buffers.changed_paths.get_or_insert_with(PrefixSetMut::default).clear(); - } else { - subtrie.buffers.changed_paths = None; - } - } - } - - /// Set whether changed node base paths should be retained during hashing. - pub fn with_changed_paths(mut self, retain_changed_paths: bool) -> Self { - self.set_changed_paths(retain_changed_paths); - self - } - - /// Takes all retained changed node base paths, preserving allocation capacity for reuse. - pub fn take_changed_paths(&mut self) -> PrefixSetMut { - match self.buffers.changed_paths.take() { - Some(changed_paths) => { - self.buffers.changed_paths = Some(PrefixSetMut::with_capacity(changed_paths.len())); - changed_paths - } - None => PrefixSetMut::default(), - } - } - /// Resets the debug recorder and records the current trie state as `SetRoot` + `RevealNodes` /// ops, representing the initial state at the beginning of a block (after pruning). /// @@ -904,10 +861,7 @@ impl ArenaParallelSparseTrie { } trace!(target: TRACE_TARGET, ?child_path, "Wrapping child into subtrie"); - let mut subtrie = ArenaSparseSubtrie::new( - self.buffers.updates.is_some(), - self.buffers.changed_paths.is_some(), - ); + let mut subtrie = ArenaSparseSubtrie::new(self.buffers.updates.is_some()); subtrie.path = *child_path; let mut root_node = mem::replace(&mut self.upper_arena[child_idx], ArenaSparseNode::TakenSubtrie); @@ -1020,10 +974,6 @@ impl ArenaParallelSparseTrie { unreachable!("recycle_subtrie called on non-Subtrie node") }; Self::merge_subtrie_updates(&mut self.buffers.updates, &mut subtrie.buffers.updates); - Self::merge_subtrie_changed_paths( - &mut self.buffers.changed_paths, - &mut subtrie.buffers.changed_paths, - ); } /// Removes a [`ArenaSparseNode::Subtrie`] from the upper arena at `idx` and recycles it. @@ -1146,10 +1096,6 @@ impl ArenaParallelSparseTrie { &mut self.buffers.updates, &mut subtrie.buffers.updates, ); - Self::merge_subtrie_changed_paths( - &mut self.buffers.changed_paths, - &mut subtrie.buffers.changed_paths, - ); // The migrated subtrie root may be a branch whose children now live in // the upper arena at or beyond the subtrie boundary depth. Re-wrap any @@ -1188,15 +1134,6 @@ impl ArenaParallelSparseTrie { } } - /// Merges changed node base paths from a subtrie's buffer into the parent's buffer. - /// Both `dst` and `src` must be `Some` when changed path tracking is enabled. - fn merge_subtrie_changed_paths(dst: &mut Option, src: &mut Option) { - if let Some(dst_changed_paths) = dst.as_mut() { - let src_changed_paths = src.as_mut().expect("changed path tracking is enabled"); - dst_changed_paths.append(src_changed_paths); - } - } - /// Right-pads a nibble path with zeros and packs it into a [`B256`]. fn nibbles_to_padded_b256(path: &Nibbles) -> B256 { let mut bytes = [0u8; 32]; @@ -1249,7 +1186,6 @@ impl ArenaParallelSparseTrie { let rlp_buf = &mut buffers.rlp_buf; let rlp_node_buf = &mut buffers.rlp_node_buf; let updates = &mut buffers.updates; - let changed_paths = &mut buffers.changed_paths; rlp_node_buf.clear(); @@ -1260,16 +1196,11 @@ impl ArenaParallelSparseTrie { ArenaSparseNode::EmptyRoot => return RlpNode::word_rlp(&EMPTY_ROOT_HASH), ArenaSparseNode::Leaf { .. } => { Self::encode_leaf(arena, root, rlp_buf, rlp_node_buf); - let was_dirty = arena[root].state_mut().take_cached_was_dirty(); - if was_dirty && let Some(changed_paths) = changed_paths.as_mut() { - changed_paths.insert(base_path); - } return rlp_node_buf.pop().expect("encode_leaf must push an RlpNode"); } ArenaSparseNode::Branch(b) => { if let ArenaSparseNodeState::Cached { rlp_node, .. } = &b.state { let rlp_node = rlp_node.clone(); - arena[root].state_mut().take_cached_was_dirty(); return rlp_node; } } @@ -1316,14 +1247,7 @@ impl ArenaParallelSparseTrie { rlp_node_buf.clear(); let state_mask = arena[head_idx].branch_ref().state_mask; - let branch_logical_path = { - let branch = arena[head_idx].branch_ref(); - let mut path = head_path; - path.extend(&branch.short_key); - path - }; - let mut child_subtree_emitted_changed_path = false; - for (child_idx, nibble) in BranchChildIter::new(state_mask) { + for (child_idx, _nibble) in BranchChildIter::new(state_mask) { match &arena[head_idx].branch_ref().children[child_idx] { ArenaSparseNodeBranchChild::Blinded(rlp_node) => { rlp_node_buf.push(rlp_node.clone()); @@ -1333,14 +1257,6 @@ impl ArenaParallelSparseTrie { match &arena[child_idx] { ArenaSparseNode::Leaf { .. } => { Self::encode_leaf(arena, child_idx, rlp_buf, rlp_node_buf); - if arena[child_idx].state_mut().take_cached_was_dirty() { - child_subtree_emitted_changed_path = true; - if let Some(changed_paths) = changed_paths.as_mut() { - let mut child_path = branch_logical_path; - child_path.push(nibble); - changed_paths.insert(child_path); - } - } } ArenaSparseNode::Branch(child_b) => { let ArenaSparseNodeState::Cached { rlp_node, .. } = &child_b.state @@ -1348,9 +1264,6 @@ impl ArenaParallelSparseTrie { panic!("child branch must be cached after DFS"); }; let rlp_node = rlp_node.clone(); - if arena[child_idx].state_mut().take_cached_was_dirty() { - child_subtree_emitted_changed_path = true; - } rlp_node_buf.push(rlp_node); } ArenaSparseNode::Subtrie(subtrie) => { @@ -1405,16 +1318,9 @@ impl ArenaParallelSparseTrie { ); let branch = arena[head_idx].branch_mut(); - branch.state = ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone(), was_dirty }; + branch.state = ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone() }; branch.branch_masks = new_branch_masks; - if was_dirty && - !child_subtree_emitted_changed_path && - let Some(changed_paths) = changed_paths.as_mut() - { - changed_paths.insert(head_path); - } - // Record trie updates for dirty branches only. // Skip the root node (empty logical path) as PST does. if let Some(trie_updates) = updates.as_mut().filter(|_| was_dirty) { @@ -1437,10 +1343,7 @@ impl ArenaParallelSparseTrie { let ArenaSparseNodeState::Cached { rlp_node, .. } = &arena[root].branch_ref().state else { panic!("root must be cached after update_cached_rlp"); }; - let rlp_node = rlp_node.clone(); - // The root has no parent to consume this one-shot marker, so clear it before returning. - arena[root].state_mut().take_cached_was_dirty(); - rlp_node + rlp_node.clone() } /// Immutable traversal to find a leaf value at `full_path` starting from `root` in `arena`. @@ -1584,13 +1487,10 @@ impl ArenaParallelSparseTrie { return; } - let was_dirty = matches!(state, ArenaSparseNodeState::Dirty); - rlp_buf.clear(); let rlp_node = LeafNodeRef { key, value }.rlp(rlp_buf); - *arena[idx].state_mut() = - ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone(), was_dirty }; + *arena[idx].state_mut() = ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone() }; rlp_node_buf.push(rlp_node); } @@ -2284,7 +2184,7 @@ impl ArenaParallelSparseTrie { let mut arena_node = ArenaSparseNode::from_proof_node(proof_node); let state = arena_node.state_mut(); - *state = ArenaSparseNodeState::Cached { rlp_node: cached_rlp, was_dirty: false }; + *state = ArenaSparseNodeState::Cached { rlp_node: cached_rlp }; let child_idx = arena.insert(arena_node); arena[head_idx].branch_mut().children[dense_child_idx] = @@ -2370,10 +2270,6 @@ impl ArenaParallelSparseTrie { } Self::merge_subtrie_updates(&mut self.buffers.updates, &mut subtrie.buffers.updates); - Self::merge_subtrie_changed_paths( - &mut self.buffers.changed_paths, - &mut subtrie.buffers.changed_paths, - ); } } @@ -2445,10 +2341,6 @@ impl SparseTrie for ArenaParallelSparseTrie { } } - fn set_changed_paths(&mut self, retain_changed_paths: bool) { - Self::set_changed_paths(self, retain_changed_paths); - } - #[instrument(level = "trace", target = TRACE_TARGET, skip_all, fields(num_nodes = nodes.len()))] fn reveal_nodes(&mut self, nodes: &mut [ProofTrieNodeV2]) -> SparseTrieResult<()> { if nodes.is_empty() { @@ -2740,10 +2632,6 @@ impl SparseTrie for ArenaParallelSparseTrie { } } - fn take_changed_paths(&mut self) -> PrefixSetMut { - Self::take_changed_paths(self) - } - #[instrument(level = "trace", target = TRACE_TARGET, skip_all)] fn wipe(&mut self) { trace!(target: TRACE_TARGET, "Wiping arena trie"); diff --git a/crates/trie/sparse/src/arena/nodes.rs b/crates/trie/sparse/src/arena/nodes.rs index a9cabd93253..70f0e0e3205 100644 --- a/crates/trie/sparse/src/arena/nodes.rs +++ b/crates/trie/sparse/src/arena/nodes.rs @@ -5,7 +5,6 @@ use super::{ use alloc::{boxed::Box, vec::Vec}; use alloy_primitives::{keccak256, B256}; use alloy_trie::{BranchNodeCompact, TrieMask}; -use core::mem; use reth_trie_common::{BranchNodeMasks, Nibbles, ProofTrieNodeV2, RlpNode, TrieNodeV2}; use smallvec::SmallVec; use strum::AsRefStr; @@ -19,9 +18,6 @@ pub(super) enum ArenaSparseNodeState { Cached { /// The cached RLP-encoded representation of the node. rlp_node: RlpNode, - /// Whether this node was dirty when its RLP was cached. This is a one-shot marker - /// consumed while retaining changed paths during parent branch encoding. - was_dirty: bool, }, /// The node has been modified and its RLP encoding needs recomputation. Dirty, @@ -40,14 +36,6 @@ impl ArenaSparseNodeState { _ => None, } } - - /// Returns and clears whether this node was dirty when its RLP was cached. - pub(super) fn take_cached_was_dirty(&mut self) -> bool { - match self { - Self::Cached { was_dirty, .. } => mem::take(was_dirty), - _ => false, - } - } } /// Represents a reference from a branch node to one of its children. diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index 58b286b1e25..21075c405b4 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -9,7 +9,7 @@ use alloy_primitives::{map::B256Map, B256}; use either::Either; use reth_execution_errors::{SparseStateTrieResult, SparseTrieErrorKind}; use reth_trie_common::{ - prefix_set::{PrefixSet, PrefixSetMut, TriePrefixSets, TriePrefixSetsMut}, + prefix_set::{PrefixSet, TriePrefixSets, TriePrefixSetsMut}, updates::{StorageTrieUpdates, TrieUpdates}, DecodedMultiProof, MultiProof, Nibbles, ProofTrieNodeV2, }; @@ -39,8 +39,6 @@ pub struct SparseStateTrie< storage: StorageTries, /// Flag indicating whether trie updates should be retained. retain_updates: bool, - /// Flag indicating whether changed node base paths should be retained. - retain_changed_paths: bool, /// Holds data that should be dropped after final state root is calculated. deferred_drops: DeferredDrops, /// Global LFU tracker for hot `(address, slot)` storage entries. @@ -62,7 +60,6 @@ where state: Default::default(), storage: Default::default(), retain_updates: false, - retain_changed_paths: false, deferred_drops: DeferredDrops::default(), hot_slots_lfu: BucketedLfu::default(), hot_accounts_lfu: BucketedLfu::default(), @@ -152,49 +149,6 @@ impl SparseStateTrie { } impl SparseStateTrie { - /// Set the retention of changed node base paths. - pub fn set_changed_paths(&mut self, retain_changed_paths: bool) { - self.retain_changed_paths = retain_changed_paths; - self.state.set_changed_paths(retain_changed_paths); - for trie in self.storage.tries.values_mut() { - trie.set_changed_paths(retain_changed_paths); - } - for trie in &mut self.storage.cleared_tries { - trie.set_changed_paths(retain_changed_paths); - } - self.storage.default_trie.set_changed_paths(retain_changed_paths); - } - - /// Set the retention of changed node base paths. - pub fn with_changed_paths(mut self, retain_changed_paths: bool) -> Self { - self.set_changed_paths(retain_changed_paths); - self - } - - /// Returns storage trie changed paths for tries that have been revealed. - fn storage_trie_changed_paths(&mut self) -> B256Map { - self.storage - .tries - .iter_mut() - .filter_map(|(address, trie)| { - let changed_paths = trie.take_changed_paths()?; - (!changed_paths.is_empty()).then_some((*address, changed_paths)) - }) - .collect() - } - - /// Returns changed paths by taking them from the revealed sparse tries. - /// - /// Returns `None` if the accounts trie is not revealed. - pub fn take_changed_paths(&mut self) -> Option { - let storage_prefix_sets = self.storage_trie_changed_paths(); - self.state.take_changed_paths().map(|account_prefix_set| TriePrefixSetsMut { - account_prefix_set, - storage_prefix_sets, - destroyed_accounts: Default::default(), - }) - } - /// Takes all debug recorders from the account trie and all revealed storage tries. /// /// Returns a vec of `(Option, TrieDebugRecorder)` where `None` is the account trie @@ -369,19 +323,14 @@ where } let retain_updates = self.retain_updates; - let retain_changed_paths = self.retain_changed_paths; #[cfg(not(feature = "std"))] let results: Vec<_> = targets .into_iter() .map(|(_, target, mut nodes)| { let result = match target { - Either::Left(trie) => { - trie.reveal_v2_proof_nodes(&mut nodes, retain_updates, retain_changed_paths) - } - Either::Right(trie) => { - trie.reveal_v2_proof_nodes(&mut nodes, retain_updates, retain_changed_paths) - } + Either::Left(trie) => trie.reveal_v2_proof_nodes(&mut nodes, retain_updates), + Either::Right(trie) => trie.reveal_v2_proof_nodes(&mut nodes, retain_updates), }; (result, nodes) }) @@ -406,16 +355,12 @@ where .entered(); let result = match target { - Either::Left(trie) => trie.reveal_v2_proof_nodes( - &mut nodes, - retain_updates, - retain_changed_paths, - ), - Either::Right(trie) => trie.reveal_v2_proof_nodes( - &mut nodes, - retain_updates, - retain_changed_paths, - ), + Either::Left(trie) => { + trie.reveal_v2_proof_nodes(&mut nodes, retain_updates) + } + Either::Right(trie) => { + trie.reveal_v2_proof_nodes(&mut nodes, retain_updates) + } }; (result, nodes) }) @@ -955,37 +900,6 @@ mod tests { ); } - #[test] - fn take_changed_paths_from_sparse_state_trie() { - let account = B256::with_last_byte(0x01); - let slot = B256::with_last_byte(0x02); - let mut sparse = SparseStateTrie::::default(); - sparse.set_accounts_trie(RevealableSparseTrie::revealed_empty()); - sparse.insert_storage_trie(account, RevealableSparseTrie::revealed_empty()); - sparse.set_changed_paths(true); - - let mut account_updates = - B256Map::from_iter([(account, LeafUpdate::Changed(vec![0x01; 32]))]); - sparse.trie_mut().update_leaves(&mut account_updates, |_, _| {}).unwrap(); - assert!(account_updates.is_empty()); - let _ = sparse.root().unwrap(); - - let mut storage_updates = B256Map::from_iter([(slot, LeafUpdate::Changed(vec![0x02; 32]))]); - sparse - .storage_trie_mut(&account) - .unwrap() - .update_leaves(&mut storage_updates, |_, _| {}) - .unwrap(); - assert!(storage_updates.is_empty()); - let _ = sparse.storage_root(&account).unwrap(); - - let changed_paths = sparse.take_changed_paths().unwrap(); - assert!(changed_paths.account_prefix_set.iter().any(|path| *path == Nibbles::default())); - assert!(changed_paths.storage_prefix_sets[&account] - .iter() - .any(|path| *path == Nibbles::default())); - } - #[test] fn prune_keeps_retained_paths_overlay_account_and_storage() { let mut sparse = SparseStateTrie::::default(); diff --git a/crates/trie/sparse/src/traits.rs b/crates/trie/sparse/src/traits.rs index c096c3ca86f..dbd36c1b2c7 100644 --- a/crates/trie/sparse/src/traits.rs +++ b/crates/trie/sparse/src/traits.rs @@ -9,9 +9,7 @@ use alloy_primitives::{ }; use alloy_trie::BranchNodeCompact; use reth_execution_errors::SparseTrieResult; -use reth_trie_common::{ - prefix_set::PrefixSetMut, BranchNodeMasks, Nibbles, ProofTrieNodeV2, TrieNodeV2, -}; +use reth_trie_common::{BranchNodeMasks, Nibbles, ProofTrieNodeV2, TrieNodeV2}; #[cfg(feature = "trie-debug")] use crate::debug_recorder::TrieDebugRecorder; @@ -77,9 +75,6 @@ pub trait SparseTrie: Sized + Debug + Send + Sync { /// * `retain_updates` - Whether to track updates fn set_updates(&mut self, retain_updates: bool); - /// Configures the trie to retain changed node base paths during hashing. - fn set_changed_paths(&mut self, retain_changed_paths: bool); - /// Reveals one or more trie nodes if they have not been revealed before. /// /// This function decodes trie nodes and inserts them into the trie structure. It handles @@ -179,13 +174,6 @@ pub trait SparseTrie: Sized + Debug + Send + Sync { /// The accumulated updates, or an empty set if updates weren't being tracked. fn take_updates(&mut self) -> SparseTrieUpdates; - /// Consumes and returns the currently accumulated changed node base paths. - /// - /// Ancestor paths may be excluded when a descendant path is already present. - /// - /// Returns an empty set if changed paths weren't being tracked. - fn take_changed_paths(&mut self) -> PrefixSetMut; - /// Removes all nodes and values from the trie, resetting it to a blank state /// with only an empty root node. This is used when a storage root is deleted. /// diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index 657950fd46f..0a5e4791900 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -4,10 +4,7 @@ use crate::{ use alloc::{borrow::Cow, boxed::Box}; use alloy_primitives::{map::B256Map, B256}; use reth_execution_errors::{SparseTrieErrorKind, SparseTrieResult}; -use reth_trie_common::{ - prefix_set::PrefixSetMut, BranchNodeMasks, Nibbles, ProofTrieNodeV2, RlpNode, TrieMask, - TrieNodeV2, -}; +use reth_trie_common::{BranchNodeMasks, Nibbles, ProofTrieNodeV2, RlpNode, TrieMask, TrieNodeV2}; /// A sparse trie that is either in a "blind" state (no nodes are revealed, root node hash is /// unknown) or in a "revealed" state (root node has been revealed and the trie can be updated). @@ -56,8 +53,7 @@ impl RevealableSparseTrie { /// If the trie is blinded, its root node is replaced with `root`. /// /// The `masks` are used to determine how the node's children are stored. - /// The retention flags control whether trie updates and changed node base paths - /// should be tracked. + /// The retention flag controls whether trie updates should be tracked. /// /// # Returns /// @@ -67,7 +63,6 @@ impl RevealableSparseTrie { root: TrieNodeV2, masks: Option, retain_updates: bool, - retain_changed_paths: bool, ) -> SparseTrieResult<&mut T> { // if `Blind`, we initialize the revealed trie with the given root node, using a // pre-allocated trie if available. @@ -79,7 +74,6 @@ impl RevealableSparseTrie { }; revealed_trie.set_root(root, masks, retain_updates)?; - revealed_trie.set_changed_paths(retain_changed_paths); *self = Self::Revealed(revealed_trie); } @@ -94,15 +88,9 @@ impl RevealableSparseTrie { &mut self, nodes: &mut [ProofTrieNodeV2], retain_updates: bool, - retain_changed_paths: bool, ) -> SparseTrieResult<()> { let trie = if let Some(root_node) = nodes.iter().find(|n| n.path.is_empty()) { - self.reveal_root( - root_node.node.clone(), - root_node.masks, - retain_updates, - retain_changed_paths, - )? + self.reveal_root(root_node.node.clone(), root_node.masks, retain_updates)? } else { self.as_revealed_mut().ok_or(SparseTrieErrorKind::Blind)? }; @@ -214,23 +202,6 @@ impl RevealableSparseTrie { Some((revealed.root(), revealed.take_updates())) } - /// Configures a revealed or retained cleared trie to collect changed node base paths. - pub fn set_changed_paths(&mut self, retain_changed_paths: bool) { - match self { - Self::Revealed(trie) | Self::Blind(Some(trie)) => { - trie.set_changed_paths(retain_changed_paths); - } - Self::Blind(None) => {} - } - } - - /// Takes changed node base paths from the revealed trie. - /// - /// Returns `None` if the trie is still blind. - pub fn take_changed_paths(&mut self) -> Option { - Some(self.as_revealed_mut()?.take_changed_paths()) - } - /// Clears this trie, setting it to a blind state. /// /// If this instance was revealed, or was itself a `Blind` with a pre-allocated diff --git a/crates/trie/sparse/tests/suite/changed_paths.rs b/crates/trie/sparse/tests/suite/changed_paths.rs deleted file mode 100644 index 31595951313..00000000000 --- a/crates/trie/sparse/tests/suite/changed_paths.rs +++ /dev/null @@ -1,127 +0,0 @@ -use super::*; - -fn key_with_prefix(bytes: &[u8]) -> B256 { - let mut key = B256::ZERO; - key.0[..bytes.len()].copy_from_slice(bytes); - key -} - -fn contains_changed_path( - changed_paths: &reth_trie_common::prefix_set::PrefixSetMut, - path: Nibbles, -) -> bool { - changed_paths.iter().any(|changed_path| *changed_path == path) -} - -pub(super) fn test_changed_paths_record_base_paths_for_branches_and_leaves( - new_trie: fn() -> T, -) { - let mut trie = new_trie(); - trie.set_changed_paths(true); - - let key_a = key_with_prefix(&[0x12, 0x34]); - let key_b = key_with_prefix(&[0x12, 0x35]); - - let mut updates = B256Map::default(); - updates.insert(key_a, LeafUpdate::Changed(vec![0x01; 64])); - updates.insert(key_b, LeafUpdate::Changed(vec![0x02; 64])); - trie.update_leaves(&mut updates, |_, _| {}).expect("insertion should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - - let changed_paths = trie.take_changed_paths(); - assert!(!contains_changed_path(&changed_paths, Nibbles::default())); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01, 0x02, 0x03, 0x04]))); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01, 0x02, 0x03, 0x05]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01, 0x02, 0x03]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::unpack(key_a))); - assert!(!contains_changed_path(&changed_paths, Nibbles::unpack(key_b))); - - let _ = trie.root(); - assert!(trie.take_changed_paths().is_empty()); -} - -pub(super) fn test_changed_paths_skip_dirty_ancestor_branch_when_descendant_changed< - T: SparseTrie, ->( - new_trie: fn() -> T, -) { - let mut trie = new_trie(); - trie.set_changed_paths(true); - - let key_a = key_with_prefix(&[0xff, 0x10]); - let key_b = key_with_prefix(&[0xff, 0x20]); - let sibling_key = key_with_prefix(&[0xe0]); - - let mut updates = B256Map::default(); - updates.insert(key_a, LeafUpdate::Changed(vec![0x01; 64])); - updates.insert(key_b, LeafUpdate::Changed(vec![0x02; 64])); - updates.insert(sibling_key, LeafUpdate::Changed(vec![0x03; 64])); - trie.update_leaves(&mut updates, |_, _| {}).expect("insertion should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - let _ = trie.take_changed_paths(); - - let mut updates = B256Map::from_iter([(key_a, LeafUpdate::Changed(vec![0x04; 64]))]); - trie.update_leaves(&mut updates, |_, _| {}).expect("update should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - - let changed_paths = trie.take_changed_paths(); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f, 0x0f, 0x01]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::default())); - - let mut updates = B256Map::from_iter([(key_b, LeafUpdate::Changed(vec![0x05; 64]))]); - trie.update_leaves(&mut updates, |_, _| {}).expect("update should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - - let changed_paths = trie.take_changed_paths(); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f, 0x0f, 0x02]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f, 0x0f, 0x01]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x0f]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::default())); -} - -pub(super) fn test_changed_paths_record_branch_after_leaf_removal( - new_trie: fn() -> T, -) { - let mut trie = new_trie(); - trie.set_changed_paths(true); - - let removed_key = key_with_prefix(&[0x12]); - let retained_key = key_with_prefix(&[0x13]); - let retained_key_b = key_with_prefix(&[0x14]); - let sibling_key = key_with_prefix(&[0xe0]); - - let mut updates = B256Map::default(); - updates.insert(removed_key, LeafUpdate::Changed(vec![0x01])); - updates.insert(retained_key, LeafUpdate::Changed(vec![0x02])); - updates.insert(retained_key_b, LeafUpdate::Changed(vec![0x03])); - updates.insert(sibling_key, LeafUpdate::Changed(vec![0x04])); - trie.update_leaves(&mut updates, |_, _| {}).expect("insertion should succeed"); - assert!(updates.is_empty()); - - let _ = trie.root(); - let _ = trie.take_changed_paths(); - - let mut removals = B256Map::default(); - removals.insert(removed_key, LeafUpdate::Changed(Vec::new())); - trie.update_leaves(&mut removals, |_, _| {}).expect("removal should succeed"); - assert!(removals.is_empty()); - - let _ = trie.root(); - - let changed_paths = trie.take_changed_paths(); - assert!(contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::from_nibbles([0x01, 0x02]))); - assert!(!contains_changed_path(&changed_paths, Nibbles::default())); - - let _ = trie.root(); - assert!(trie.take_changed_paths().is_empty()); -} diff --git a/crates/trie/sparse/tests/suite/main.rs b/crates/trie/sparse/tests/suite/main.rs index 75ea9c3c102..a2cbc5d1d41 100644 --- a/crates/trie/sparse/tests/suite/main.rs +++ b/crates/trie/sparse/tests/suite/main.rs @@ -11,7 +11,6 @@ //! - [`update_leaves`]: Tests for `update_leaves`, including insert, modify, and remove //! - [`root`]: Tests for `root()` hash computation //! - [`take_updates`]: Tests for `take_updates` -//! - [`changed_paths`]: Tests for retained changed node base paths //! - [`prune`]: Tests for `prune` //! - [`wipe_clear`]: Tests for `wipe` and `clear` //! - [`get_leaf_value`]: Tests for `get_leaf_value` @@ -27,7 +26,6 @@ use reth_trie_common::{Nibbles, ProofV2Target, TrieNodeV2}; use reth_trie_sparse::{LeafLookup, LeafLookupError, LeafUpdate, SparseTrie}; use std::{collections::BTreeMap, iter::once}; -mod changed_paths; mod find_leaf; mod get_leaf_value; mod lifecycle; @@ -191,7 +189,6 @@ macro_rules! sparse_trie_tests { // Re-export test functions from submodules for the macro // --------------------------------------------------------------------------- -use changed_paths::*; use find_leaf::*; use get_leaf_value::*; use lifecycle::*; @@ -275,10 +272,6 @@ sparse_trie_tests! { test_take_updates_no_duplicate_updated_and_removed_nodes, test_take_updates_cross_cancellation_across_root_calls, - // changed_paths - test_changed_paths_record_base_paths_for_branches_and_leaves, - test_changed_paths_skip_dirty_ancestor_branch_when_descendant_changed, - test_changed_paths_record_branch_after_leaf_removal, // prune test_prune_retains_specified_leaves, From 268efd6cddcb415f74a54e5b39e576c428eae4ac Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 15 Jul 2026 13:16:19 +0200 Subject: [PATCH 004/148] perf(engine): skip proof worker cursor overlays in most cases (#25952) --- .../chain-state/src/preserved_sparse_trie.rs | 220 +++++++++----- crates/chain-state/src/state_trie_overlay.rs | 287 ++++++++---------- .../tree/src/tree/state_root_strategy/mod.rs | 187 +++++++++--- .../provider/src/providers/state/overlay.rs | 81 ++++- 4 files changed, 496 insertions(+), 279 deletions(-) diff --git a/crates/chain-state/src/preserved_sparse_trie.rs b/crates/chain-state/src/preserved_sparse_trie.rs index 507ea6f5c85..2d36422283d 100644 --- a/crates/chain-state/src/preserved_sparse_trie.rs +++ b/crates/chain-state/src/preserved_sparse_trie.rs @@ -2,100 +2,170 @@ use alloy_primitives::B256; use reth_trie_sparse::SparseStateTrie; +use std::{ + fmt, + sync::mpsc::{self, Receiver, Sender}, +}; use tracing::debug; /// Type alias for the sparse trie type used in preservation. pub type SparseTrie = SparseStateTrie; -/// Guard that holds the lock on the preserved trie. -/// While held, the next trie take will block. Call `store()` to save the trie before dropping. -#[derive(Debug)] -pub struct PreservedTrieGuard<'a>(parking_lot::MutexGuard<'a, Option>); - -impl<'a> PreservedTrieGuard<'a> { - /// Creates a new guard from the preserved trie lock. - pub(crate) const fn new( - guard: parking_lot::MutexGuard<'a, Option>, - ) -> Self { - PreservedTrieGuard(guard) - } - - /// Stores a preserved trie for later reuse. - pub fn store(&mut self, trie: PreservedSparseTrie) { - self.0.replace(trie); - } +/// A preserved sparse trie that can be reused across payload validations. +pub struct PreservedSparseTrie { + /// The preserved sparse state trie, or a handle to wait for it. + trie: PreservedSparseTrieInner, + /// The state root this trie represents. + /// + /// Used to verify continuity: a new payload's `parent_state_root` must match this before the + /// existing sparse trie nodes can be reused. + state_root: B256, + /// Parent block hash of the earliest overlay state covered by this trie. + anchor_hash: B256, } -/// A preserved sparse trie that can be reused across payload validations. -/// -/// The trie exists in one of two states: -/// - **Anchored**: Has a computed state root and can be reused for payloads whose parent state root -/// matches the anchor. -/// - **Cleared**: Trie data has been cleared but allocations are preserved for reuse. -#[derive(Debug)] -pub enum PreservedSparseTrie { - /// Trie with a computed state root that can be reused for continuation payloads. - Anchored { - /// The sparse state trie anchored to the computed state root. - trie: SparseTrie, - /// The state root this trie represents (computed from the previous block). - /// Used to verify continuity: new payload's `parent_state_root` must match this. - state_root: B256, - }, - /// Cleared trie with preserved allocations, ready for fresh use. - Cleared { - /// The sparse state trie with cleared data but preserved allocations. - trie: SparseTrie, - }, +impl fmt::Debug for PreservedSparseTrie { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("PreservedSparseTrie") + .field("state_root", &self.state_root) + .field("anchor_hash", &self.anchor_hash) + .finish_non_exhaustive() + } } impl PreservedSparseTrie { /// Creates a new anchored preserved trie. /// - /// The `state_root` is the computed state root from the trie, which becomes the - /// anchor for determining if subsequent payloads can reuse this trie. - pub const fn anchored(trie: SparseTrie, state_root: B256) -> Self { - Self::Anchored { trie, state_root } + /// The `state_root` is the computed state root from the trie. The `anchor_hash` is the parent + /// block hash of the earliest overlay state covered by the trie. + pub const fn anchored(trie: SparseTrie, state_root: B256, anchor_hash: B256) -> Self { + Self { trie: PreservedSparseTrieInner::Ready(trie), state_root, anchor_hash } + } + + /// Creates a pending preserved trie and a completer that will publish the trie later. + pub fn pending(state_root: B256, anchor_hash: B256) -> (Self, PreservedSparseTrieCompleter) { + let (tx, rx) = mpsc::channel(); + ( + Self { trie: PreservedSparseTrieInner::Pending(rx), state_root, anchor_hash }, + PreservedSparseTrieCompleter { tx }, + ) } - /// Creates a cleared preserved trie (allocations preserved, data cleared). - pub const fn cleared(trie: SparseTrie) -> Self { - Self::Cleared { trie } + /// Returns the state root this trie is anchored to. + pub const fn state_root(&self) -> B256 { + self.state_root } - /// Consumes self and returns the trie for reuse. + /// Returns the parent block hash of the earliest overlay state covered by this trie. + pub const fn anchor_hash(&self) -> B256 { + self.anchor_hash + } + + /// Consumes self and returns the trie if it can be reused for the parent state root. /// - /// If the preserved trie is anchored and the parent state root matches, the preserved - /// trie structure is reused directly. Otherwise, the trie is cleared but allocations - /// are preserved to reduce memory overhead. - pub fn into_trie_for(self, parent_state_root: B256) -> SparseTrie { + /// If the parent state root does not match the preserved trie's state root, this drops the trie + /// and returns `None` so the caller can create a fresh sparse trie. + pub fn into_trie_for( + self, + parent_state_root: B256, + ) -> Result, PreservedSparseTrieError> { + if self.state_root == parent_state_root { + let trie = match self.trie { + PreservedSparseTrieInner::Ready(trie) => trie, + PreservedSparseTrieInner::Pending(rx) => match rx.recv() { + Ok(trie) => trie, + Err(_) => { + return Err(PreservedSparseTrieError::ProducerDropped { + state_root: self.state_root, + }) + } + }, + }; + debug!( + target: "engine::tree::payload_processor", + state_root = %self.state_root, + anchor_hash = %self.anchor_hash, + "Reusing anchored sparse trie for continuation payload" + ); + Ok(Some(trie)) + } else { + debug!( + target: "engine::tree::payload_processor", + anchor_root = %self.state_root, + anchor_hash = %self.anchor_hash, + %parent_state_root, + "Dropping anchored sparse trie - parent state root mismatch" + ); + Ok(None) + } + } +} + +/// Error returned when consuming a preserved sparse trie. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PreservedSparseTrieError { + /// The producer of a pending preserved sparse trie dropped before publishing it. + ProducerDropped { + /// The state root the pending trie was expected to represent. + state_root: B256, + }, +} + +impl fmt::Display for PreservedSparseTrieError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { - Self::Anchored { trie, state_root } if state_root == parent_state_root => { - debug!( - target: "engine::tree::payload_processor", - %state_root, - "Reusing anchored sparse trie for continuation payload" - ); - trie - } - Self::Anchored { mut trie, state_root } => { - debug!( - target: "engine::tree::payload_processor", - anchor_root = %state_root, - %parent_state_root, - "Clearing anchored sparse trie - parent state root mismatch" - ); - trie.clear(); - trie - } - Self::Cleared { trie } => { - debug!( - target: "engine::tree::payload_processor", - %parent_state_root, - "Using cleared sparse trie with preserved allocations" - ); - trie + Self::ProducerDropped { state_root } => { + write!(f, "pending preserved sparse trie producer dropped for {state_root}") } } } } + +impl std::error::Error for PreservedSparseTrieError {} + +#[allow(clippy::large_enum_variant)] +enum PreservedSparseTrieInner { + Ready(SparseTrie), + Pending(Receiver), +} + +/// Completes a pending preserved sparse trie. +#[derive(Debug)] +pub struct PreservedSparseTrieCompleter { + tx: Sender, +} + +impl PreservedSparseTrieCompleter { + /// Publishes the trie for a pending preserved sparse trie. + pub fn complete(self, trie: SparseTrie) -> Result<(), SparseTrie> { + self.tx.send(trie).map_err(|err| err.0) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn pending_trie_exposes_state_root_before_completion() { + let state_root = B256::with_last_byte(1); + let anchor_hash = B256::with_last_byte(2); + let (preserved, completer) = PreservedSparseTrie::pending(state_root, anchor_hash); + + assert_eq!(preserved.state_root(), state_root); + assert_eq!(preserved.anchor_hash(), anchor_hash); + completer.complete(SparseTrie::default()).unwrap(); + assert!(preserved.into_trie_for(state_root).unwrap().is_some()); + } + + #[test] + fn pending_trie_with_mismatched_root_does_not_wait() { + let state_root = B256::with_last_byte(1); + let other_state_root = B256::with_last_byte(2); + let anchor_hash = B256::with_last_byte(3); + let (preserved, completer) = PreservedSparseTrie::pending(state_root, anchor_hash); + + assert!(preserved.into_trie_for(other_state_root).unwrap().is_none()); + assert!(completer.complete(SparseTrie::default()).is_err()); + } +} diff --git a/crates/chain-state/src/state_trie_overlay.rs b/crates/chain-state/src/state_trie_overlay.rs index 369599e7b29..a56e1f2a693 100644 --- a/crates/chain-state/src/state_trie_overlay.rs +++ b/crates/chain-state/src/state_trie_overlay.rs @@ -4,7 +4,7 @@ //! parent has not been persisted yet. [`StateTrieOverlayManager`] tracks those in-memory blocks and //! builds reusable flattened state trie overlays on demand. -use crate::{EthPrimitives, ExecutedBlock, PreservedSparseTrie, PreservedTrieGuard}; +use crate::{EthPrimitives, ExecutedBlock, PreservedSparseTrie}; use alloy_primitives::B256; use parking_lot::Mutex; use reth_metrics::{ @@ -86,17 +86,19 @@ impl StateTrieOverlayManager { } } - /// Takes the preserved sparse trie if present, leaving `None` in its place. + /// Takes the preserved sparse trie if present. pub fn take_sparse_trie(&self) -> Option { self.preserved_sparse_trie.lock().take() } - /// Acquires a guard that blocks taking the trie until dropped. - /// - /// Use this before sending the state root result to ensure the next block waits for the trie - /// to be stored. - pub fn lock_sparse_trie(&self) -> PreservedTrieGuard<'_> { - PreservedTrieGuard::new(self.preserved_sparse_trie.lock()) + /// Stores a preserved sparse trie for later reuse. + pub fn store_sparse_trie(&self, trie: PreservedSparseTrie) { + *self.preserved_sparse_trie.lock() = Some(trie); + } + + /// Clears any preserved sparse trie state. + pub fn clear_sparse_trie(&self) { + *self.preserved_sparse_trie.lock() = None; } /// Waits until the sparse trie lock becomes available. @@ -150,54 +152,12 @@ impl StateTrieOverlayManager { } } - // Snapshot matching parent overlays before spawning so DashMap iteration guards are - // dropped. - let cached_parent_overlays = self - .overlays - .iter() - .filter_map(|entry| { - let key = *entry.key(); - (key.tip_hash == parent_hash && entry.value().is_ready()).then_some(key.anchor_hash) - }) - .collect::>(); - debug!( target: "chain_state::state_trie_overlay", %hash, %parent_hash, "inserted block into state trie overlay manager" ); - if cached_parent_overlays.is_empty() { - return - } - - #[cfg(feature = "rayon")] - let Some(worker_pool) = self.worker_pool.clone() else { - return - }; - - #[cfg(not(feature = "rayon"))] - let _ = cached_parent_overlays; - - #[cfg(feature = "rayon")] - { - let parent_span = span; - for anchor_hash in cached_parent_overlays { - let manager = ::clone(self); - let parent_span = parent_span.clone(); - worker_pool.spawn(move || { - let _span = tracing::trace_span!( - target: "chain_state::state_trie_overlay", - parent: parent_span, - "precompute_state_trie_overlay", - tip_hash = %hash, - anchor_hash = %anchor_hash, - ) - .entered(); - let _ = manager.precompute_overlay(hash, anchor_hash); - }); - } - } } /// Removes blocks from the live block graph and prunes cached overlays that can no longer be @@ -268,16 +228,6 @@ impl StateTrieOverlayManager { Ok((Arc::clone(&input.nodes), Arc::clone(&input.state))) } - #[cfg(feature = "rayon")] - fn precompute_overlay( - &self, - tip_hash: B256, - anchor_hash: B256, - ) -> Result<(), StateTrieOverlayError> { - let _ = self.get_overlay_inner(tip_hash, anchor_hash, OverlayLookupMode::Precompute)?; - Ok(()) - } - #[tracing::instrument( level = "trace", target = "chain_state::state_trie_overlay", @@ -295,35 +245,14 @@ impl StateTrieOverlayManager { tip_hash: B256, anchor_hash: B256, ) -> Result, StateTrieOverlayError> { - self.get_overlay_inner(tip_hash, anchor_hash, OverlayLookupMode::Required) - .map(|input| input.expect("required overlay lookups always return an overlay")) - } - - fn get_overlay_inner( - &self, - tip_hash: B256, - anchor_hash: B256, - mode: OverlayLookupMode, - ) -> Result>, StateTrieOverlayError> { let key = OverlayCacheKey { anchor_hash, tip_hash }; let span = tracing::Span::current(); if let Some(entry) = self.overlays.get(&key).map(|entry| entry.value().clone()) { + self.record_overlay_cache_reuse(&span); return Ok(match entry { - OverlayCacheEntry::Ready(input) => { - self.metrics.overlay_cache_reuses.increment(1); - span.record("cache_reused", true); - Some(input) - } - OverlayCacheEntry::Computing(waiter) => { - span.record("cache_reused", true); - if mode == OverlayLookupMode::Precompute { - None - } else { - self.metrics.overlay_cache_reuses.increment(1); - Some(waiter.wait()) - } - } + OverlayCacheEntry::Ready(input) => input, + OverlayCacheEntry::Computing(waiter) => waiter.wait(), }) } span.record("cache_reused", false); @@ -365,26 +294,17 @@ impl StateTrieOverlayManager { Ready(Arc), Wait(Arc), Compute(Arc), - Skip, } let action = match self.overlays.entry(key) { - Entry::Occupied(entry) => match entry.get().clone() { - OverlayCacheEntry::Ready(input) => { - self.metrics.overlay_cache_reuses.increment(1); - span.record("cache_reused", true); - CacheAction::Ready(input) - } - OverlayCacheEntry::Computing(waiter) => { - span.record("cache_reused", true); - if mode == OverlayLookupMode::Precompute { - CacheAction::Skip - } else { - self.metrics.overlay_cache_reuses.increment(1); - CacheAction::Wait(waiter) - } + Entry::Occupied(entry) => { + let entry = entry.get().clone(); + self.record_overlay_cache_reuse(&span); + match entry { + OverlayCacheEntry::Ready(input) => CacheAction::Ready(input), + OverlayCacheEntry::Computing(waiter) => CacheAction::Wait(waiter), } - }, + } Entry::Vacant(entry) => { self.metrics.overlay_cache_fills.increment(1); let waiter = Arc::new(OverlayWaiter::new()); @@ -394,9 +314,8 @@ impl StateTrieOverlayManager { }; match action { - CacheAction::Ready(input) => Ok(Some(input)), - CacheAction::Wait(waiter) => Ok(Some(waiter.wait())), - CacheAction::Skip => Ok(None), + CacheAction::Ready(input) => Ok(input), + CacheAction::Wait(waiter) => Ok(waiter.wait()), CacheAction::Compute(waiter) => { let input = self.compute_overlay(compute_input, anchor_hash, span); waiter.finish(Arc::clone(&input)); @@ -413,11 +332,16 @@ impl StateTrieOverlayManager { } } - Ok(Some(input)) + Ok(input) } } } + fn record_overlay_cache_reuse(&self, span: &tracing::Span) { + self.metrics.overlay_cache_reuses.increment(1); + span.record("cache_reused", true); + } + /// Returns `preferred_anchor` if it is on the parent chain, otherwise the first missing parent. /// /// Returns `None` if `parent_hash` is not `preferred_anchor` and the manager does not contain a @@ -426,6 +350,24 @@ impl StateTrieOverlayManager { Self::anchor_for_parent_in(self.blocks.as_ref(), parent_hash, preferred_anchor) } + /// Returns true if `hash` is in the parent chain segment from `anchor_hash` inclusive to + /// `parent_hash` inclusive. + pub fn contains_hash(&self, parent_hash: B256, anchor_hash: B256, hash: B256) -> bool { + let mut current_hash = parent_hash; + + loop { + if current_hash == hash { + return true + } + if current_hash == anchor_hash { + return false + } + + let Some(block) = self.blocks.get(¤t_hash) else { return false }; + current_hash = block.recovered_block().parent_hash(); + } + } + fn anchor_for_parent_in( blocks: &DashMap>, parent_hash: B256, @@ -505,10 +447,6 @@ enum OverlayCacheEntry { } impl OverlayCacheEntry { - const fn is_ready(&self) -> bool { - matches!(self, Self::Ready(_)) - } - fn ready(&self) -> Option> { match self { Self::Ready(input) => Some(Arc::clone(input)), @@ -517,12 +455,6 @@ impl OverlayCacheEntry { } } -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum OverlayLookupMode { - Required, - Precompute, -} - struct OverlayWaiter { input: OnceLock>, } @@ -673,12 +605,10 @@ fn extend_overlay( #[cfg(test)] mod tests { use super::*; - use crate::{test_utils::TestBlockBuilder, EthPrimitives, ExecutedBlock}; + use crate::{test_utils::TestBlockBuilder, EthPrimitives, ExecutedBlock, SparseTrie}; use alloy_primitives::U256; use reth_primitives_traits::Account; use reth_trie::{updates::TrieUpdatesSorted, ComputedTrieData, HashedPostState, HashedStorage}; - #[cfg(feature = "rayon")] - use std::time::Instant; use std::{ sync::{mpsc, Arc}, thread, @@ -787,6 +717,79 @@ mod tests { ); } + #[test] + fn contains_hash_detects_hashes_from_anchor_to_parent() { + let manager = StateTrieOverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let anchor_hash = blocks[0].recovered_block().parent_hash(); + let parent_hash = blocks[2].recovered_block().hash(); + + assert!(manager.contains_hash(parent_hash, anchor_hash, anchor_hash)); + for block in &blocks { + assert!(manager.contains_hash( + parent_hash, + anchor_hash, + block.recovered_block().hash() + )); + } + assert!(!manager.contains_hash(parent_hash, anchor_hash, B256::random())); + } + + #[test] + fn contains_hash_rejects_hash_before_anchor() { + let manager = StateTrieOverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let parent_hash = blocks[2].recovered_block().hash(); + let anchor_hash = blocks[1].recovered_block().hash(); + let before_anchor_hash = blocks[0].recovered_block().hash(); + + assert!(manager.contains_hash(parent_hash, anchor_hash, parent_hash)); + assert!(manager.contains_hash(parent_hash, anchor_hash, anchor_hash)); + assert!(!manager.contains_hash(parent_hash, anchor_hash, before_anchor_hash)); + } + + #[test] + fn contains_hash_rejects_unknown_anchor() { + let manager = StateTrieOverlayManager::default(); + let blocks = test_blocks(); + for block in &blocks { + manager.insert_block(block.clone()); + } + + let parent_hash = blocks[2].recovered_block().hash(); + let anchor_hash = B256::random(); + + assert!(!manager.contains_hash(parent_hash, anchor_hash, anchor_hash)); + } + + #[test] + fn taking_sparse_trie_removes_it() { + let manager = StateTrieOverlayManager::::default(); + let state_root = B256::with_last_byte(1); + let other_state_root = B256::with_last_byte(2); + let anchor_hash = B256::with_last_byte(3); + + manager.store_sparse_trie(PreservedSparseTrie::anchored( + SparseTrie::default(), + state_root, + anchor_hash, + )); + + let preserved = manager.take_sparse_trie().expect("preserved trie should be available"); + assert_eq!(preserved.state_root(), state_root); + assert_eq!(preserved.anchor_hash(), anchor_hash); + assert!(preserved.into_trie_for(other_state_root).unwrap().is_none()); + assert!(manager.take_sparse_trie().is_none()); + } + #[test] fn required_lookup_waits_for_in_progress_overlay() { let manager = StateTrieOverlayManager::::default(); @@ -815,56 +818,6 @@ mod tests { assert!(state.is_empty()); } - #[cfg(feature = "rayon")] - #[test] - fn precompute_skips_in_progress_overlay() { - let manager = StateTrieOverlayManager::::new(Arc::new(WorkerPool::new( - 1, - "test-ovly", - ))); - let key = OverlayCacheKey { - anchor_hash: B256::with_last_byte(1), - tip_hash: B256::with_last_byte(2), - }; - manager.overlays.insert(key, OverlayCacheEntry::Computing(Arc::new(OverlayWaiter::new()))); - - let (tx, rx) = mpsc::channel(); - thread::spawn(move || { - tx.send(manager.precompute_overlay(key.tip_hash, key.anchor_hash)).unwrap(); - }); - - rx.recv_timeout(Duration::from_secs(1)).unwrap().unwrap(); - } - - #[cfg(feature = "rayon")] - #[test] - fn insert_block_prepares_child_overlay_from_cached_parent() { - let manager = StateTrieOverlayManager::new(Arc::new(WorkerPool::new(2, "test-ovly"))); - let blocks = test_blocks(); - - manager.insert_block(blocks[0].clone()); - - let anchor_hash = blocks[0].recovered_block().parent_hash(); - let parent_hash = blocks[0].recovered_block().hash(); - manager.overlay_for_parent(parent_hash, anchor_hash).unwrap(); - - let child_hash = blocks[1].recovered_block().hash(); - manager.insert_block(blocks[1].clone()); - - let child_key = OverlayCacheKey { anchor_hash, tip_hash: child_hash }; - let deadline = Instant::now() + Duration::from_secs(5); - while !manager.overlays.contains_key(&child_key) { - assert!( - Instant::now() < deadline, - "timed out waiting for optimistically prepared child overlay" - ); - thread::sleep(Duration::from_millis(10)); - } - - let (_, state) = manager.overlay_for_parent(child_hash, anchor_hash).unwrap(); - assert_eq!(state.accounts.len(), 2); - } - #[test] fn prunes_cached_overlays_after_removing_blocks() { let manager = StateTrieOverlayManager::default(); diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 3893bb5214b..7db42af1e35 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -512,7 +512,9 @@ impl DefaultStateRootStrategy { + 'static, { let StateRootTaskOptions { + parent_hash, parent_state_root, + preserved_sparse_trie, transaction_count, config, pending_sparse_trie_prune_blocks, @@ -539,7 +541,9 @@ impl DefaultStateRootStrategy { from_multi_proof, cancel_rx, SparseTrieTaskOptions { + parent_hash, parent_state_root, + preserved_sparse_trie, chunk_size: config.multiproof_chunk_size(), pending_sparse_trie_prune_blocks: if config.disable_sparse_trie_cache_pruning() { None @@ -574,7 +578,9 @@ impl DefaultStateRootStrategy { options: SparseTrieTaskOptions, ) { let SparseTrieTaskOptions { + parent_hash, parent_state_root, + preserved_sparse_trie, chunk_size, pending_sparse_trie_prune_blocks, max_hot_slots, @@ -595,26 +601,46 @@ impl DefaultStateRootStrategy { ) .entered(); - let start = Instant::now(); - let preserved = state_trie_overlays.take_sparse_trie(); - trie_metrics - .sparse_trie_cache_wait_duration_histogram - .record(start.elapsed().as_secs_f64()); - - let mut sparse_state_trie = preserved - .map(|preserved| preserved.into_trie_for(parent_state_root)) - .unwrap_or_else(|| { - debug!( - target: "engine::tree::payload_processor", - "Creating new sparse trie - no preserved trie available" - ); - let default_trie = - RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); - SparseStateTrie::default() - .with_accounts_trie(default_trie.clone()) - .with_default_storage_trie(default_trie) - .with_updates(true) - }); + let new_sparse_state_trie = || { + debug!( + target: "engine::tree::payload_processor", + "Creating new sparse trie - no preserved trie available" + ); + let default_trie = + RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); + SparseStateTrie::default() + .with_accounts_trie(default_trie.clone()) + .with_default_storage_trie(default_trie) + .with_updates(true) + }; + + let mut sparse_trie_anchor_hash = parent_hash; + let mut reused_preserved_sparse_trie = false; + let mut sparse_state_trie = match preserved_sparse_trie { + Some(preserved) => { + let start = Instant::now(); + let preserved_anchor_hash = preserved.anchor_hash(); + let preserved = preserved.into_trie_for(parent_state_root); + trie_metrics + .sparse_trie_cache_wait_duration_histogram + .record(start.elapsed().as_secs_f64()); + + match preserved { + Ok(Some(trie)) => { + sparse_trie_anchor_hash = preserved_anchor_hash; + reused_preserved_sparse_trie = true; + trie + } + Ok(None) => new_sparse_state_trie(), + Err(err) => { + let _ = + state_root_tx.send(Err(StateRootTaskError::Other(err.to_string()))); + return; + } + } + } + None => new_sparse_state_trie(), + }; sparse_state_trie.set_hot_cache_capacities(max_hot_slots, max_hot_accounts); let mut task = SparseTrieCacheTask::new_with_trie( @@ -630,27 +656,48 @@ impl DefaultStateRootStrategy { ); let result = task.run(); - // Lock before publishing the result. The next block can start as soon as the receiver - // wakes and must not observe an empty overlay before this task stores the reusable - // trie. - let mut guard = state_trie_overlays.lock_sparse_trie(); let task_result = result.as_ref().ok().cloned(); + // Publish a handle before sending the result so the next block can inspect the + // state root immediately while the trie is finalized for reuse below. + let pending_trie = if let Some(result) = &task_result { + let preserved_anchor_hash = published_sparse_trie_anchor_hash( + sparse_trie_anchor_hash, + reused_preserved_sparse_trie, + pending_sparse_trie_prune_blocks.as_deref(), + ); + let (preserved, completer) = + PreservedSparseTrie::pending(result.state_root, preserved_anchor_hash); + state_trie_overlays.store_sparse_trie(preserved); + Some(completer) + } else { + state_trie_overlays.clear_sparse_trie(); + None + }; + if state_root_tx.send(result).is_err() { + // A continuation task can take the pending trie during the narrow window between + // publishing it and detecting the abandoned receiver here. Returning drops the + // completer, so the taker wakes with `ProducerDropped` and its state-root consumer + // falls back to serial computation. No partially finalized trie is exposed; the + // worst case is a redundant fallback. debug!( target: "engine::tree::payload_processor", - "State root receiver dropped, clearing trie" + "State root receiver dropped, dropping trie" ); let (trie, deferred) = task.into_cleared_trie(); - guard.store(PreservedSparseTrie::cleared(trie)); - drop(guard); + state_trie_overlays.clear_sparse_trie(); + executor.spawn_drop(trie); executor.spawn_drop(deferred); - return + return; } let _enter = debug_span!(target: "engine::tree::payload_processor", "preserve").entered(); + let mut trie_to_drop = None; let deferred = if let Some(result) = task_result { + let pending_trie = + pending_trie.expect("pending trie is created for successful task result"); let start = Instant::now(); let (mut trie, deferred) = task.into_trie_for_reuse(); if let Some(prune_blocks) = pending_sparse_trie_prune_blocks { @@ -665,25 +712,31 @@ impl DefaultStateRootStrategy { trie_metrics .sparse_trie_retained_storage_tries .set(trie.retained_storage_tries_count() as f64); - guard.store(PreservedSparseTrie::anchored(trie, result.state_root)); + if let Err(trie) = pending_trie.complete(trie) { + trie_to_drop = Some(trie); + } deferred } else { debug!( target: "engine::tree::payload_processor", - "State root computation failed, clearing trie" + "State root computation failed, dropping trie" ); let (trie, deferred) = task.into_cleared_trie(); - guard.store(PreservedSparseTrie::cleared(trie)); + trie_to_drop = Some(trie); deferred }; - drop(guard); + if let Some(trie) = trie_to_drop { + executor.spawn_drop(trie); + } executor.spawn_drop(deferred); }); } } struct SparseTrieTaskOptions { + parent_hash: B256, parent_state_root: B256, + preserved_sparse_trie: Option, chunk_size: usize, /// `None` disables pruning. `Some(Vec::new())` prunes using only the current block's paths. pending_sparse_trie_prune_blocks: Option>>, @@ -692,7 +745,9 @@ struct SparseTrieTaskOptions { } struct StateRootTaskOptions<'a, N: NodePrimitives> { + parent_hash: B256, parent_state_root: B256, + preserved_sparse_trie: Option, transaction_count: Option, config: &'a TreeConfig, pending_sparse_trie_prune_blocks: Option>>, @@ -711,6 +766,20 @@ fn sparse_trie_retained_paths( retained_paths } +fn published_sparse_trie_anchor_hash( + sparse_trie_anchor_hash: B256, + reused_preserved_sparse_trie: bool, + pending_sparse_trie_prune_blocks: Option<&[ExecutedBlock]>, +) -> B256 { + if !reused_preserved_sparse_trie { + return sparse_trie_anchor_hash + } + + pending_sparse_trie_prune_blocks + .and_then(|blocks| blocks.last().map(|block| block.recovered_block().parent_hash())) + .unwrap_or(sparse_trie_anchor_hash) +} + impl StateRootStrategy for DefaultStateRootStrategy where N: NodePrimitives, @@ -754,12 +823,25 @@ where state: _, } = ctx; + let preserved_sparse_trie = state_trie_overlays.take_sparse_trie(); + let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie + .as_ref() + .filter(|trie| trie.state_root() == env.parent_state_root) + .map(|trie| trie.anchor_hash()) + { + overlay_factory.with_skip_overlay_for_reused_sparse_trie(anchor_hash) + } else { + overlay_factory + }; + let mut handle = self.spawn_state_root( executor, state_trie_overlays, overlay_factory.clone(), StateRootTaskOptions { + parent_hash: env.parent_hash, parent_state_root: env.parent_state_root, + preserved_sparse_trie, transaction_count: Some(env.transaction_count), config, pending_sparse_trie_prune_blocks, @@ -816,13 +898,26 @@ where } let pending_sparse_trie_prune_blocks = ctx.take_sparse_trie_prune_blocks(); + let parent_state_root = ctx.parent_state_root(); + let preserved_sparse_trie = ctx.state_trie_overlays.take_sparse_trie(); + let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie + .as_ref() + .filter(|trie| trie.state_root() == parent_state_root) + .map(|trie| trie.anchor_hash()) + { + ctx.overlay_factory.clone().with_skip_overlay_for_reused_sparse_trie(anchor_hash) + } else { + ctx.overlay_factory.clone() + }; Ok(Some( self.spawn_state_root( ctx.executor, ctx.state_trie_overlays, - ctx.overlay_factory.clone(), + overlay_factory, StateRootTaskOptions { - parent_state_root: ctx.parent_state_root(), + parent_hash: ctx.parent_hash(), + parent_state_root, + preserved_sparse_trie, // Tx count unknown at FCU time (block built incrementally): full proof workers. transaction_count: None, config: ctx.config, @@ -1262,6 +1357,28 @@ mod tests { assert_eq!(retained_paths.storage_prefix_sets.len(), 2); } + #[test] + fn published_sparse_trie_anchor_uses_prune_anchor_for_reused_trie() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect(); + let reused_anchor_hash = B256::with_last_byte(0xaa); + + assert_eq!( + published_sparse_trie_anchor_hash(reused_anchor_hash, true, Some(&blocks)), + blocks.last().unwrap().recovered_block().parent_hash() + ); + } + + #[test] + fn published_sparse_trie_anchor_keeps_parent_for_fresh_trie() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect(); + let parent_hash = B256::with_last_byte(0xaa); + + assert_eq!( + published_sparse_trie_anchor_hash(parent_hash, false, Some(&blocks)), + parent_hash + ); + } + fn create_mock_state_updates(num_accounts: usize, updates_per_account: usize) -> Vec { let mut rng = generators::rng(); let all_addresses: Vec
= (0..num_accounts).map(|_| rng.random()).collect(); @@ -1365,7 +1482,9 @@ mod tests { OverlayBuilder::::new(genesis_hash, ChangesetCache::new()), ), StateRootTaskOptions { + parent_hash: genesis_hash, parent_state_root: env.parent_state_root, + preserved_sparse_trie: None, transaction_count: Some(env.transaction_count), config: &TreeConfig::default(), pending_sparse_trie_prune_blocks: None, diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index d9373603e5b..6d7b88db881 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -52,6 +52,9 @@ pub(crate) struct OverlayStateProviderMetrics { database_provider_ro_duration: Histogram, /// Number of cache misses when fetching [`Overlay`]s from the overlay cache. overlay_cache_misses: Counter, + /// Number of managed overlay creations skipped because the reused sparse trie already covers + /// the DB tip to parent range. + sparse_trie_overlay_skips: Counter, } /// Contains all fields required to initialize an [`OverlayStateProvider`]. @@ -61,6 +64,15 @@ pub(super) struct Overlay { pub(super) hashed_post_state: Arc, } +impl Overlay { + fn empty() -> Self { + Self { + trie_updates: Arc::new(TrieUpdatesSorted::default()), + hashed_post_state: Arc::new(HashedPostStateSorted::default()), + } + } +} + /// Source of overlay data for [`OverlayStateProviderFactory`]. #[derive(Debug, Clone)] pub(super) enum OverlaySource { @@ -97,6 +109,8 @@ pub struct OverlayBuilder { overlay_source: Option>, /// Changeset cache handle for retrieving trie changesets changeset_cache: ChangesetCache, + /// Anchor hash of the reused sparse trie, if this task reused one. + reused_sparse_trie_anchor_hash: Option, /// Metrics for tracking provider operations metrics: OverlayStateProviderMetrics, } @@ -108,6 +122,7 @@ impl OverlayBuilder { parent_hash, overlay_source: None, changeset_cache, + reused_sparse_trie_anchor_hash: None, metrics: OverlayStateProviderMetrics::default(), } } @@ -120,6 +135,13 @@ impl OverlayBuilder { self } + /// Skips managed overlay construction when the sparse trie was reused and the DB tip is + /// already covered by its anchor-to-parent range. + pub const fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { + self.reused_sparse_trie_anchor_hash = Some(anchor_hash); + self + } + /// Set the state trie overlay manager used to resolve in-memory parent state. pub fn with_state_trie_overlay_manager( mut self, @@ -213,6 +235,18 @@ impl OverlayBuilder { } } + /// Returns true if managed overlay resolution can be skipped for this builder. + fn should_skip_overlay_for_reused_sparse_trie(&self, db_tip_hash: B256) -> bool { + let Some(anchor_hash) = self.reused_sparse_trie_anchor_hash else { return false }; + + match &self.overlay_source { + Some(OverlaySource::Managed { manager, state }) if state.is_empty() => { + manager.contains_hash(self.parent_hash, anchor_hash, db_tip_hash) + } + _ => false, + } + } + /// Returns the block which is at the tip of the DB, i.e. the block which the state tables of /// the DB are currently synced to. fn get_db_tip_block(&self, provider: &Provider) -> ProviderResult @@ -395,14 +429,28 @@ impl OverlayBuilder { (trie_updates, hashed_state_updates) } else { // If no reverts are needed then the db tip is the anchor hash. Use overlays directly. - let (trie_updates, hashed_state) = self.resolve_overlays(db_tip_block.hash)?; + if self.should_skip_overlay_for_reused_sparse_trie(db_tip_block.hash) { + debug!( + target: "providers::state::overlay", + parent_hash = %self.parent_hash, + db_tip_hash = %db_tip_block.hash, + sparse_trie_anchor_hash = ?self.reused_sparse_trie_anchor_hash, + "Skipping overlay construction because reused sparse trie covers DB tip to parent" + ); + + self.metrics.sparse_trie_overlay_skips.increment(1); + + return Ok(Overlay::empty()) + } + + let (trie_updates, hashed_post_state) = self.resolve_overlays(db_tip_block.hash)?; retrieve_trie_reverts_duration = Duration::ZERO; retrieve_hashed_state_reverts_duration = Duration::ZERO; trie_updates_total_len = trie_updates.total_len(); - hashed_state_updates_total_len = hashed_state.total_len(); + hashed_state_updates_total_len = hashed_post_state.total_len(); - (trie_updates, hashed_state) + (trie_updates, hashed_post_state) }; // Record metrics @@ -466,6 +514,15 @@ impl OverlayStateProviderFactory { self } + /// Skips managed overlay construction when this factory is used by a task that reused a sparse + /// trie covering the DB tip to parent range. + pub fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { + self.overlay_builder = + self.overlay_builder.with_skip_overlay_for_reused_sparse_trie(anchor_hash); + self.overlay_cache = Default::default(); + self + } + /// Extends the existing hashed state overlay with the given [`HashedPostStateSorted`]. pub fn with_extended_hashed_state_overlay(mut self, other: HashedPostStateSorted) -> Self { self.overlay_builder = self.overlay_builder.with_extended_hashed_state_overlay(other); @@ -699,4 +756,22 @@ mod tests { }; assert_eq!(state.total_len(), 1); } + + #[test] + fn managed_overlay_skip_requires_reused_sparse_trie_and_no_immediate_state() { + let parent_hash = B256::with_last_byte(1); + let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) + .with_state_trie_overlay_manager(StateTrieOverlayManager::default()); + assert!(!builder.should_skip_overlay_for_reused_sparse_trie(parent_hash)); + + let builder = builder.with_skip_overlay_for_reused_sparse_trie(parent_hash); + assert!(builder.should_skip_overlay_for_reused_sparse_trie(parent_hash)); + assert!(!builder.should_skip_overlay_for_reused_sparse_trie(B256::with_last_byte(3))); + + let hashed_state = HashedPostState::default() + .with_accounts([(B256::with_last_byte(2), Some(Account::default()))]) + .into_sorted(); + let builder = builder.with_extended_hashed_state_overlay(hashed_state); + assert!(!builder.should_skip_overlay_for_reused_sparse_trie(parent_hash)); + } } From 4ed6acaffdf0823cfef0f512fc7319b332710c3d Mon Sep 17 00:00:00 2001 From: William Law Date: Wed, 15 Jul 2026 08:24:31 -0400 Subject: [PATCH 005/148] feat(download): make env public and add getter (#25285) --- crates/cli/commands/src/download/mod.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/crates/cli/commands/src/download/mod.rs b/crates/cli/commands/src/download/mod.rs index 5cedbe4ad7e..f0b1fd7e52c 100644 --- a/crates/cli/commands/src/download/mod.rs +++ b/crates/cli/commands/src/download/mod.rs @@ -994,6 +994,11 @@ where } impl DownloadCommand { + /// Returns a reference to the environment arguments. + pub const fn env(&self) -> &EnvironmentArgs { + &self.env + } + /// Returns the underlying chain being used to run this command pub fn chain_spec(&self) -> Option<&Arc> { Some(&self.env.chain) From 2bcf7acc426d6bcf7a84e0170005fca8b178ffe5 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 16 Jul 2026 13:54:54 +0300 Subject: [PATCH 006/148] fix: terminate eyre bail invocations (#26416) --- crates/cli/commands/src/config_cmd.rs | 4 +++- crates/cli/commands/src/download/archive.rs | 2 +- crates/cli/commands/src/download/manifest.rs | 2 +- crates/cli/commands/src/download/manifest_cmd.rs | 2 +- crates/cli/commands/src/p2p/bootnode.rs | 8 ++++++-- crates/cli/commands/src/p2p/mod.rs | 4 ++-- crates/cli/commands/src/stage/unwind.rs | 2 +- crates/engine/local/src/miner.rs | 8 ++++---- crates/node/builder/src/launch/invalid_block_hook.rs | 2 +- crates/node/core/src/utils.rs | 4 ++-- crates/storage/db-common/src/db_tool/mod.rs | 2 +- 11 files changed, 23 insertions(+), 17 deletions(-) diff --git a/crates/cli/commands/src/config_cmd.rs b/crates/cli/commands/src/config_cmd.rs index e12f468fac9..571c2087139 100644 --- a/crates/cli/commands/src/config_cmd.rs +++ b/crates/cli/commands/src/config_cmd.rs @@ -24,7 +24,9 @@ impl Command { } else { let path = match self.config.as_ref() { Some(path) => path, - None => bail!("No config file provided. Use --config or pass --default"), + None => { + bail!("No config file provided. Use --config or pass --default"); + } }; if !path.exists() { bail!("Config file does not exist: {}", path.display()); diff --git a/crates/cli/commands/src/download/archive.rs b/crates/cli/commands/src/download/archive.rs index 05ae98166cf..f1189486f29 100644 --- a/crates/cli/commands/src/download/archive.rs +++ b/crates/cli/commands/src/download/archive.rs @@ -203,7 +203,7 @@ impl ArchiveProcessor { "Failed integrity validation after {} attempts for {}", MAX_DOWNLOAD_RETRIES, archive.file_name - ) + ); } } } diff --git a/crates/cli/commands/src/download/manifest.rs b/crates/cli/commands/src/download/manifest.rs index 430f1eeb0d1..cc3f3e1adf9 100644 --- a/crates/cli/commands/src/download/manifest.rs +++ b/crates/cli/commands/src/download/manifest.rs @@ -742,7 +742,7 @@ fn state_source_files(source_datadir: &Path) -> Result> { return collect_files_recursive(source_datadir, Path::new("db")); } - eyre::bail!("Could not find source state DB directory under {}", source_datadir.display()) + eyre::bail!("Could not find source state DB directory under {}", source_datadir.display()); } fn rocksdb_source_files(source_datadir: &Path) -> Result> { diff --git a/crates/cli/commands/src/download/manifest_cmd.rs b/crates/cli/commands/src/download/manifest_cmd.rs index ab34abd2f5b..25e2d0398de 100644 --- a/crates/cli/commands/src/download/manifest_cmd.rs +++ b/crates/cli/commands/src/download/manifest_cmd.rs @@ -126,7 +126,7 @@ fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result eyre::bail!( "Could not infer --block from source DB (Finish checkpoint missing); pass --block manually" - ) + ); } /// Infers the snapshot block from the highest header static-file range. diff --git a/crates/cli/commands/src/p2p/bootnode.rs b/crates/cli/commands/src/p2p/bootnode.rs index 03c98855cfe..cacc6f041ed 100644 --- a/crates/cli/commands/src/p2p/bootnode.rs +++ b/crates/cli/commands/src/p2p/bootnode.rs @@ -202,7 +202,9 @@ impl Command { advertised_ips: vec![first_ip, second_ip], }) } - _ => eyre::bail!("--nat can be provided at most twice"), + _ => { + eyre::bail!("--nat can be provided at most twice"); + } } } @@ -283,7 +285,9 @@ async fn bind_socket(addr: SocketAddr) -> eyre::Result> { fn fixed_external_ip(nat: &NatResolver) -> eyre::Result { match nat { NatResolver::ExternalIp(ip) => Ok(*ip), - _ => eyre::bail!("--nat can only be repeated with extip: values"), + _ => { + eyre::bail!("--nat can only be repeated with extip: values"); + } } } diff --git a/crates/cli/commands/src/p2p/mod.rs b/crates/cli/commands/src/p2p/mod.rs index 8f29fb70cb1..d1f1a83bee8 100644 --- a/crates/cli/commands/src/p2p/mod.rs +++ b/crates/cli/commands/src/p2p/mod.rs @@ -76,7 +76,7 @@ impl eyre::bail!( "Invalid number of bodies received. Expected: 1. Received: {}", result.len() - ) + ); } let body = result.into_iter().next().unwrap(); tracing::info!(target: "reth::cli", ?body, "Successfully downloaded body") @@ -185,7 +185,7 @@ impl DownloadArgs { if config.peers.trusted_nodes.is_empty() && self.network.trusted_only { eyre::bail!( "No trusted nodes. Set trusted peer with `--trusted-peer ` or set `--trusted-only` to `false`" - ) + ); } config.peers.trusted_nodes_only |= self.network.trusted_only; diff --git a/crates/cli/commands/src/stage/unwind.rs b/crates/cli/commands/src/stage/unwind.rs index 2fdda790a0d..b3f6f072c80 100644 --- a/crates/cli/commands/src/stage/unwind.rs +++ b/crates/cli/commands/src/stage/unwind.rs @@ -178,7 +178,7 @@ impl Subcommands { if target > last { eyre::bail!( "Target block number {target} is higher than the latest block number {last}" - ) + ); } Ok(target) } diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index ff131b35152..37d88d6a551 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -227,7 +227,7 @@ where let res = self.to_engine.fork_choice_updated(state, None).await?; if !res.is_valid() { - eyre::bail!("Invalid fork choice update {state:?}: {res:?}") + eyre::bail!("Invalid fork choice update {state:?}: {res:?}"); } Ok(()) @@ -245,7 +245,7 @@ where .await?; if !res.is_valid() { - eyre::bail!("Invalid payload status") + eyre::bail!("Invalid payload status"); } let payload_id = res.payload_id.ok_or_eyre("No payload id")?; @@ -257,14 +257,14 @@ where let Some(Ok(payload)) = self.payload_builder.resolve_kind(payload_id, PayloadKind::WaitForPending).await else { - eyre::bail!("No payload") + eyre::bail!("No payload"); }; let header = payload.block().sealed_header().clone(); let res = self.to_engine.new_payload(payload.into()).await?; if !res.is_valid() { - eyre::bail!("Invalid payload") + eyre::bail!("Invalid payload"); } self.last_block_hashes.push_back(header.hash()); diff --git a/crates/node/builder/src/launch/invalid_block_hook.rs b/crates/node/builder/src/launch/invalid_block_hook.rs index 3c1848dceb4..7965634c55b 100644 --- a/crates/node/builder/src/launch/invalid_block_hook.rs +++ b/crates/node/builder/src/launch/invalid_block_hook.rs @@ -105,7 +105,7 @@ where healthy_node_rpc_client.clone(), )), InvalidBlockHookType::PreState | InvalidBlockHookType::Opcode => { - eyre::bail!("invalid block hook {hook:?} is not implemented yet") + eyre::bail!("invalid block hook {hook:?} is not implemented yet"); } } as Box>) }) diff --git a/crates/node/core/src/utils.rs b/crates/node/core/src/utils.rs index 68cab7929f3..e63337c2735 100644 --- a/crates/node/core/src/utils.rs +++ b/crates/node/core/src/utils.rs @@ -41,7 +41,7 @@ where let Some(header) = response else { client.report_bad_message(peer_id); - eyre::bail!("Invalid number of headers received. Expected: 1. Received: 0") + eyre::bail!("Invalid number of headers received. Expected: 1. Received: 0"); }; let header = SealedHeader::seal_slow(header); @@ -77,7 +77,7 @@ where let Some(body) = response else { client.report_bad_message(peer_id); - eyre::bail!("Invalid number of bodies received. Expected: 1. Received: 0") + eyre::bail!("Invalid number of bodies received. Expected: 1. Received: 0"); }; let block = SealedBlock::from_sealed_parts(header, body); diff --git a/crates/storage/db-common/src/db_tool/mod.rs b/crates/storage/db-common/src/db_tool/mod.rs index e9d7f81b0f6..4f7c7fe2c28 100644 --- a/crates/storage/db-common/src/db_tool/mod.rs +++ b/crates/storage/db-common/src/db_tool/mod.rs @@ -36,7 +36,7 @@ impl DbTool { pub fn list(&self, filter: &ListFilter) -> Result<(Vec>, usize)> { let bmb = Rc::new(BMByte::from(&filter.search)); if bmb.is_none() && filter.has_search() { - eyre::bail!("Invalid search.") + eyre::bail!("Invalid search."); } let mut hits = 0; From 0ac3d4100a8729b2360db9fd1f9508dc8170c6da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Thu, 16 Jul 2026 16:51:38 +0200 Subject: [PATCH 007/148] feat(net): serve real snap/2 account and storage range data (#26339) --- Cargo.lock | 1 + crates/cli/commands/src/stage/run.rs | 8 +- crates/net/network/Cargo.toml | 3 +- crates/net/network/src/config.rs | 3 + crates/net/network/src/eth_requests.rs | 682 +++++++++++++++++- crates/net/network/src/test_utils/testnet.rs | 8 +- .../src/providers/blockchain_provider.rs | 584 ++++++++++++++- .../provider/src/providers/database/mod.rs | 5 + .../provider/src/providers/state/overlay.rs | 35 +- .../storage/provider/src/test_utils/mock.rs | 216 +++++- crates/storage/provider/src/traits/full.rs | 6 +- crates/storage/storage-api/src/noop.rs | 11 +- crates/storage/storage-api/src/trie.rs | 81 ++- 13 files changed, 1596 insertions(+), 47 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d5c880ba9bd..bc0144c7b1c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9094,6 +9094,7 @@ dependencies = [ "serde", "smallvec", "socket2", + "test-case", "thiserror 2.0.18", "tokio", "tokio-stream", diff --git a/crates/cli/commands/src/stage/run.rs b/crates/cli/commands/src/stage/run.rs index c8601a3cf4b..c99946219d7 100644 --- a/crates/cli/commands/src/stage/run.rs +++ b/crates/cli/commands/src/stage/run.rs @@ -30,8 +30,8 @@ use reth_node_metrics::{ }; use reth_primitives_traits::FastInstant as Instant; use reth_provider::{ - ChainSpecProvider, DBProvider, DatabaseProviderFactory, StageCheckpointReader, - StageCheckpointWriter, + providers::BlockchainProvider, ChainSpecProvider, DBProvider, DatabaseProviderFactory, + StageCheckpointReader, StageCheckpointWriter, }; use reth_stages::{ stages::{ @@ -175,7 +175,7 @@ impl default_peers_path, runtime.clone(), ) - .build(provider_factory.clone()) + .build(BlockchainProvider::new(provider_factory.clone())?) .start_network() .await?; let fetch_client = Arc::new(network.fetch_client().await?); @@ -231,7 +231,7 @@ impl default_peers_path, runtime.clone(), ) - .build(provider_factory.clone()) + .build(BlockchainProvider::new(provider_factory.clone())?) .start_network() .await?; let fetch_client = Arc::new(network.fetch_client().await?); diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index 298d93abced..951d720b51c 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -39,7 +39,7 @@ reth-network-types = { workspace = true, features = ["serde"] } alloy-consensus.workspace = true alloy-eips.workspace = true alloy-primitives.workspace = true -alloy-rlp.workspace = true +alloy-rlp = { workspace = true, features = ["derive"] } enr = { workspace = true, features = ["serde", "rust-secp256k1"] } discv5.workspace = true @@ -95,6 +95,7 @@ reth-transaction-pool = { workspace = true, features = ["test-utils"] } alloy-genesis.workspace = true # misc +test-case.workspace = true url.workspace = true secp256k1 = { workspace = true, features = ["rand"] } diff --git a/crates/net/network/src/config.rs b/crates/net/network/src/config.rs index 470dd841d00..7f106b9c176 100644 --- a/crates/net/network/src/config.rs +++ b/crates/net/network/src/config.rs @@ -22,6 +22,7 @@ use reth_network_peers::{mainnet_nodes, pk2id, sepolia_nodes, PeerId, TrustedPee use reth_network_types::{PeersConfig, SessionsConfig}; use reth_storage_api::{ noop::NoopProvider, BalProvider, BlockNumReader, BlockReader, HeaderProvider, + StateProviderFactory, StateRangeProviderFactory, }; use reth_tasks::Runtime; use secp256k1::SECP256K1; @@ -163,6 +164,8 @@ impl NetworkConfig where N: NetworkPrimitives, C: BalProvider + + StateProviderFactory + + StateRangeProviderFactory + BlockReader + HeaderProvider + Clone diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index c6d4dee8f08..89068af2c93 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -4,12 +4,20 @@ use crate::{ budget::DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS, metered_poll_nested_stream_with_budget, metrics::EthRequestHandlerMetrics, }; -use alloy_consensus::{BlockHeader, ReceiptWithBloom}; +use alloy_consensus::{ + constants::{EMPTY_ROOT_HASH, KECCAK_EMPTY}, + BlockHeader, ReceiptWithBloom, +}; use alloy_eips::BlockHashOrNumber; -use alloy_rlp::Encodable; +use alloy_primitives::{Bytes, B256, U256}; +use alloy_rlp::{Encodable, RlpEncodable}; use futures::StreamExt; use reth_eth_wire::{ - snap::{BlockAccessListsMessage, SnapProtocolMessage}, + snap::{ + AccountData, AccountRangeMessage, BlockAccessListsMessage, ByteCodesMessage, + GetAccountRangeMessage, GetStorageRangesMessage, SnapProtocolMessage, StorageData, + StorageRangesMessage, + }, BlockAccessLists, BlockBodies, BlockHeaders, Cells, EthNetworkPrimitives, GetBlockAccessLists, GetBlockBodies, GetBlockHeaders, GetCells, GetNodeData, GetReceipts, GetReceipts70, HeadersDirection, NetworkPrimitives, NodeData, Receipts, Receipts69, Receipts70, @@ -20,8 +28,12 @@ use reth_network_p2p::{ snap::client::SnapResponse, }; use reth_network_peers::PeerId; -use reth_primitives_traits::Block; -use reth_storage_api::{BalProvider, BlockReader, GetBlockAccessListLimit, HeaderProvider}; +use reth_primitives_traits::{Account, Block}; +use reth_storage_api::{ + errors::provider::ProviderResult, BalProvider, BlockReader, BytecodeReader, + GetBlockAccessListLimit, HeaderProvider, RangeEnd, RangeResponse, StateProviderFactory, + StateRangeProviderFactory, +}; use reth_transaction_pool::{blobstore::NoopBlobStore, BlobStore}; use std::{ future::Future, @@ -60,6 +72,14 @@ pub const MAX_BLOCK_ACCESS_LISTS_SERVE: usize = 1024; /// Used to limit lookups. pub const MAX_CELLS_SERVE: usize = 1024; +/// Maximum number of bytecode lookups to serve. +/// +/// Used to limit lookups. +pub const MAX_BYTE_CODES_SERVE: usize = 1024; + +/// Maximum number of storage range account lookups to serve. +pub const MAX_STORAGE_RANGE_ACCOUNTS_SERVE: usize = 1024; + /// Maximum size of replies to data retrievals: 2MB pub const SOFT_RESPONSE_LIMIT: usize = 2 * 1024 * 1024; @@ -383,14 +403,20 @@ where self.client.bal_store().get_by_hashes_with_limit(&request.0, limit).unwrap_or_default(); let _ = response.send(Ok(BlockAccessLists(access_lists))); } +} +impl EthRequestHandler +where + N: NetworkPrimitives, + C: BalProvider + StateProviderFactory + StateRangeProviderFactory, +{ /// Handles `snap/2` (EIP-8189) requests. /// - /// `GetAccountRange`/`GetStorageRanges`/`GetByteCodes` stay unsupported until a real - /// state-trie-backed store exists; an empty response would falsely claim served data. + /// `GetAccountRange`/`GetStorageRanges` are hash-native throughout and served from retained + /// canonical roots via [`StateRangeProviderFactory`]. `GetByteCodes` is content-addressed and + /// independent of any particular state root, so it's served directly. /// `GetBlockAccessLists` is answered from the same [`BalProvider`] store eth71's - /// `GetBlockAccessLists` uses, since both serve the same - /// underlying data. + /// `GetBlockAccessLists` uses, since both serve the same underlying data. fn on_snap_request( &self, _peer_id: PeerId, @@ -400,9 +426,31 @@ where self.metrics.snap_requests_received_total.increment(1); let result = match request { - SnapProtocolMessage::GetAccountRange(_) | - SnapProtocolMessage::GetStorageRanges(_) | - SnapProtocolMessage::GetByteCodes(_) => Err(RequestError::UnsupportedCapability), + SnapProtocolMessage::GetAccountRange(req) => { + let request_id = req.request_id; + let response = self.get_account_range_response(req).unwrap_or_else(|error| { + tracing::debug!(target: "net::snap", %error, "failed to serve account range"); + AccountRangeMessage { request_id, accounts: Vec::new(), proof: Vec::new() } + }); + Ok(SnapResponse::AccountRange(response)) + } + SnapProtocolMessage::GetStorageRanges(req) => { + let request_id = req.request_id; + let response = self.get_storage_ranges_response(req).unwrap_or_else(|error| { + tracing::debug!(target: "net::snap", %error, "failed to serve storage ranges"); + StorageRangesMessage { request_id, slots: Vec::new(), proof: Vec::new() } + }); + Ok(SnapResponse::StorageRanges(response)) + } + SnapProtocolMessage::GetByteCodes(req) => { + let codes = self + .get_byte_codes_response(&req.hashes, req.response_bytes as usize) + .unwrap_or_else(|error| { + tracing::debug!(target: "net::snap", %error, "failed to serve bytecodes"); + Vec::new() + }); + Ok(SnapResponse::ByteCodes(ByteCodesMessage { request_id: req.request_id, codes })) + } SnapProtocolMessage::GetBlockAccessLists(mut req) => { req.block_hashes.truncate(MAX_BLOCK_ACCESS_LISTS_SERVE); let limit = GetBlockAccessListLimit::ResponseSizeSoftLimit( @@ -425,6 +473,173 @@ where let _ = response.send(result); } + + /// Returns the bytecode for each of `hashes`, skipping hashes with no known code, stopping + /// once `response_bytes` (capped at [`SOFT_RESPONSE_LIMIT`]) is exceeded. + fn get_byte_codes_response( + &self, + hashes: &[B256], + response_bytes: usize, + ) -> ProviderResult> { + let state = self.client.latest()?; + let response_bytes = response_bytes.min(SOFT_RESPONSE_LIMIT); + + let mut codes = Vec::new(); + let mut total_bytes = 0; + for hash in hashes.iter().take(MAX_BYTE_CODES_SERVE) { + let bytes = if *hash == KECCAK_EMPTY { + Bytes::new() + } else { + match state.bytecode_by_hash(hash)? { + Some(bytecode) => bytecode.original_bytes(), + None => continue, + } + }; + total_bytes += bytes.len(); + codes.push(bytes); + + if total_bytes > response_bytes { + break + } + } + Ok(codes) + } + + /// Serves a `GetAccountRange` request via [`StateRangeProviderFactory`]. + /// + /// Fails the request if a storage root or proof becomes unavailable after the range lookup. + /// Always proves the boundary between `starting_hash` and the last returned account, per + /// snap/2's boundary-proof requirement. + fn get_account_range_response( + &self, + req: GetAccountRangeMessage, + ) -> ProviderResult { + let empty = AccountRangeMessage { + request_id: req.request_id, + accounts: Vec::new(), + proof: Vec::new(), + }; + + let response_bytes = (req.response_bytes as usize).min(SOFT_RESPONSE_LIMIT); + let Some(state) = self.client.state_range_provider(req.root_hash)? else { + return Ok(empty) + }; + let RangeResponse { items: accounts, .. } = + state.account_range(req.starting_hash, req.limit_hash, response_bytes)?; + + let boundary_keys = boundary_proof_keys(req.starting_hash, accounts.last()); + + let mut account_data = Vec::with_capacity(accounts.len()); + for (hash, account) in accounts { + let storage_root = state.storage_root_by_hash(hash)?; + account_data + .push(AccountData { hash, body: slim_account_body(&account, storage_root) }); + } + + let proof = state.account_range_proof(&boundary_keys)?; + + Ok(AccountRangeMessage { request_id: req.request_id, accounts: account_data, proof }) + } + + /// Serves a `GetStorageRanges` request via [`StateRangeProviderFactory`]. + /// + /// `starting_hash`/`limit_hash` apply only to the first account. An account unavailable at + /// this root makes the whole response empty, rather than skipping it and shifting later + /// accounts' positions. A proof stops the response at the first account that isn't a + /// complete, zero-origin range. + fn get_storage_ranges_response( + &self, + req: GetStorageRangesMessage, + ) -> ProviderResult { + let empty = StorageRangesMessage { + request_id: req.request_id, + slots: Vec::new(), + proof: Vec::new(), + }; + let Some(state) = self.client.state_range_provider(req.root_hash)? else { + return Ok(empty) + }; + let mut slots = Vec::new(); + let mut proof = Vec::new(); + let mut remaining_bytes = (req.response_bytes as usize).min(SOFT_RESPONSE_LIMIT); + + for (i, &hashed_address) in + req.account_hashes.iter().take(MAX_STORAGE_RANGE_ACCOUNTS_SERVE).enumerate() + { + if remaining_bytes == 0 { + break + } + let origin = if i == 0 { req.starting_hash } else { B256::ZERO }; + let limit = if i == 0 { req.limit_hash } else { B256::repeat_byte(0xff) }; + let Some(RangeResponse { items: account_slots, end }) = + state.storage_range(hashed_address, origin, limit, remaining_bytes)? + else { + return Ok(empty) + }; + + remaining_bytes = remaining_bytes.saturating_sub(account_slots.len() * 64); + let last = account_slots.last().map(|(hash, _)| *hash); + let needs_proof = origin != B256::ZERO || end != RangeEnd::Exhausted; + slots.push( + account_slots + .into_iter() + .map(|(hash, value)| StorageData { + hash, + // snap clients verify proofs against RLP-encoded storage trie leaves. + data: alloy_rlp::encode(value).into(), + }) + .collect(), + ); + + if needs_proof { + let boundary_keys = match last { + Some(last) => vec![origin, last], + None => vec![origin], + }; + proof = state.storage_range_proof(hashed_address, &boundary_keys)?; + break + } + } + + Ok(StorageRangesMessage { request_id: req.request_id, slots, proof }) + } +} + +/// Boundary-proof keys for a range reply: `origin`, plus the last returned item's key if any. +fn boundary_proof_keys(origin: B256, last: Option<&(B256, T)>) -> Vec { + match last { + Some((last, _)) => vec![origin, *last], + None => vec![origin], + } +} + +/// Like the consensus trie account, but the code hash and storage root are empty byte strings +/// rather than [`KECCAK_EMPTY`]/[`EMPTY_ROOT_HASH`] when the account has no code/storage, to +/// avoid transferring the same 32 bytes for every EOA. +#[derive(RlpEncodable)] +struct SlimAccountBody<'a> { + nonce: u64, + balance: U256, + storage_root: &'a [u8], + code_hash: &'a [u8], +} + +/// RLP-encodes `account` in snap/2's slim format; see [`SlimAccountBody`]. +fn slim_account_body(account: &Account, storage_root: B256) -> Bytes { + let storage_root: &[u8] = + if storage_root == EMPTY_ROOT_HASH { &[] } else { storage_root.as_slice() }; + let code_hash: &[u8] = match &account.bytecode_hash { + Some(hash) if *hash != KECCAK_EMPTY => hash.as_slice(), + _ => &[], + }; + + alloy_rlp::encode(SlimAccountBody { + nonce: account.nonce, + balance: account.balance, + storage_root, + code_hash, + }) + .into() } /// An endless future. @@ -434,6 +649,8 @@ impl Future for EthRequestHandler where N: NetworkPrimitives, C: BalProvider + + StateProviderFactory + + StateRangeProviderFactory + BlockReader + HeaderProvider
+ Unpin, @@ -606,14 +823,16 @@ mod tests { eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobTransactionSidecarVariant, Cell}, }; - use alloy_primitives::{TxHash, B128, B256}; + use alloy_primitives::{keccak256, Address, TxHash, B128}; use reth_network_api::test_utils::PeersHandle; + use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_storage_api::noop::NoopProvider; use reth_transaction_pool::blobstore::{BlobStoreCleanupStat, BlobStoreError}; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, }; + use test_case::test_case; use tokio::sync::mpsc; #[derive(Debug, Default)] @@ -754,4 +973,441 @@ mod tests { assert!(cells.hashes.is_empty()); assert_eq!(get_cells_calls.load(Ordering::Relaxed), MAX_CELLS_SERVE); } + + /// Creates a request handler backed by the mock provider for snap response tests. + fn snap_handler( + provider: MockEthProvider, + ) -> EthRequestHandler { + let (peers_tx, _) = mpsc::unbounded_channel(); + let (_incoming_tx, incoming_rx) = mpsc::channel(1); + EthRequestHandler::new(provider, PeersHandle::new(peers_tx), incoming_rx) + } + + #[test_case( + SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 1, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: Vec::new(), + proof: Vec::new(), + }); "account range" + )] + #[test_case( + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 2, + root_hash: B256::ZERO, + account_hashes: vec![B256::ZERO], + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 2, + slots: Vec::new(), + proof: Vec::new(), + }); "storage ranges" + )] + #[test_case( + SnapProtocolMessage::GetByteCodes(reth_eth_wire::snap::GetByteCodesMessage { + request_id: 3, + hashes: vec![B256::repeat_byte(0x11)], + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + SnapResponse::ByteCodes(ByteCodesMessage { request_id: 3, codes: Vec::new() }); "bytecodes" + )] + #[tokio::test] + async fn snap_requests_return_empty_responses_on_provider_errors( + request: SnapProtocolMessage, + expected: SnapResponse, + ) { + let provider = MockEthProvider::default(); + provider.set_snap_state_reads_fail(true); + let handler = snap_handler(provider); + let (response, rx) = oneshot::channel(); + + handler.on_snap_request(PeerId::default(), request, response); + + assert_eq!(rx.await.unwrap(), Ok(expected)); + } + + #[tokio::test] + async fn unavailable_snap_state_returns_empty_response() { + let provider = MockEthProvider::default(); + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 1, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: Vec::new(), + proof: Vec::new(), + })) + ); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_requests_return_empty_responses_on_inconsistent_provider_results() { + let missing_storage_root = MockEthProvider::default(); + missing_storage_root + .set_snap_account_range(vec![(B256::ZERO, Account::default())], RangeEnd::Exhausted); + + let missing_account_proof = MockEthProvider::default(); + missing_account_proof.set_snap_account_range(Vec::new(), RangeEnd::Exhausted); + + let missing_storage_proof = MockEthProvider::default(); + missing_storage_proof + .push_snap_storage_range(vec![(B256::ZERO, U256::from(1))], RangeEnd::ByteLimit); + + let storage_disappears = MockEthProvider::default(); + storage_disappears.push_snap_storage_range(Vec::new(), RangeEnd::Exhausted); + storage_disappears.push_unavailable_snap_storage_range(); + + let account_request = || { + SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 1, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }) + }; + let storage_request = |account_hashes| { + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 2, + root_hash: B256::ZERO, + account_hashes, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }) + }; + let empty_accounts = SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: Vec::new(), + proof: Vec::new(), + }); + let empty_storage = SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 2, + slots: Vec::new(), + proof: Vec::new(), + }); + let cases = [ + (missing_storage_root, account_request(), empty_accounts.clone()), + (missing_account_proof, account_request(), empty_accounts), + (missing_storage_proof, storage_request(vec![B256::ZERO]), empty_storage.clone()), + (storage_disappears, storage_request(vec![B256::ZERO, B256::ZERO]), empty_storage), + ]; + + for (provider, request, expected) in cases { + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request(PeerId::default(), request, response); + assert_eq!(rx.await.unwrap(), Ok(expected)); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + } + + #[tokio::test] + async fn snap_account_range_response_encodes_accounts_and_proof() { + let provider = MockEthProvider::default(); + let first_hash = B256::repeat_byte(0x01); + let second_hash = B256::repeat_byte(0x02); + let storage_root = B256::repeat_byte(0x11); + let code_hash = B256::repeat_byte(0x22); + let proof = vec![Bytes::from_static(&[0xaa])]; + provider.set_snap_account_range( + vec![ + ( + first_hash, + Account { nonce: 1, balance: U256::from(2), bytecode_hash: Some(code_hash) }, + ), + (second_hash, Account { nonce: 3, balance: U256::from(4), bytecode_hash: None }), + ], + RangeEnd::Exhausted, + ); + provider.set_snap_storage_root(first_hash, storage_root); + provider.set_snap_storage_root(second_hash, EMPTY_ROOT_HASH); + provider.set_snap_account_proof(Some(proof.clone())); + + let mut full_body = vec![0xf8, 0x44, 0x01, 0x02, 0xa0]; + full_body.extend_from_slice(storage_root.as_slice()); + full_body.push(0xa0); + full_body.extend_from_slice(code_hash.as_slice()); + let empty_body = Bytes::from_static(&[0xc4, 0x03, 0x04, 0x80, 0x80]); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage { + request_id: 1, + root_hash: B256::ZERO, + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::AccountRange(AccountRangeMessage { + request_id: 1, + accounts: vec![ + AccountData { hash: first_hash, body: full_body.into() }, + AccountData { hash: second_hash, body: empty_body }, + ], + proof, + })) + ); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_storage_range_response_encodes_values_and_proof() { + let provider = MockEthProvider::default(); + let first_hash = B256::repeat_byte(0x01); + let second_hash = B256::repeat_byte(0x02); + let origin = B256::repeat_byte(0x10); + let proof = vec![Bytes::from_static(&[0xbb])]; + provider.push_snap_storage_range( + vec![(first_hash, U256::from(0x0102)), (second_hash, U256::from(0xff))], + RangeEnd::Exhausted, + ); + provider.set_snap_storage_proof(Some(proof.clone())); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 2, + root_hash: B256::ZERO, + account_hashes: vec![B256::repeat_byte(0x03)], + starting_hash: origin, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 2, + slots: vec![vec![ + StorageData { hash: first_hash, data: Bytes::from_static(&[0x82, 0x01, 0x02]) }, + StorageData { hash: second_hash, data: Bytes::from_static(&[0x81, 0xff]) }, + ]], + proof, + })) + ); + assert_eq!(provider.snap_storage_range_requests()[0].1, origin); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_storage_ranges_only_bound_the_first_account() { + let provider = MockEthProvider::default(); + provider.push_snap_storage_range(Vec::new(), RangeEnd::Exhausted); + provider.push_snap_storage_range(Vec::new(), RangeEnd::Exhausted); + let first_account = B256::repeat_byte(0x01); + let second_account = B256::repeat_byte(0x02); + let origin = B256::ZERO; + let limit = B256::repeat_byte(0x22); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 3, + root_hash: B256::ZERO, + account_hashes: vec![first_account, second_account], + starting_hash: origin, + limit_hash: limit, + response_bytes: 1_000, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 3, + slots: vec![Vec::new(), Vec::new()], + proof: Vec::new(), + })) + ); + assert_eq!( + provider.snap_storage_range_requests(), + vec![ + (first_account, origin, limit, 1_000), + (second_account, B256::ZERO, B256::repeat_byte(0xff), 1_000), + ] + ); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_byte_codes_response_preserves_found_code_order() { + let provider = MockEthProvider::default(); + let code = Bytes::from_static(&[0x60, 0x00]); + let code_hash = keccak256(&code); + let later_code = Bytes::from_static(&[0x60, 0x01]); + let later_code_hash = keccak256(&later_code); + provider.add_account( + Address::repeat_byte(0x01), + ExtendedAccount::new(1, U256::ZERO).with_bytecode(code.clone()), + ); + provider.add_account( + Address::repeat_byte(0x02), + ExtendedAccount::new(1, U256::ZERO).with_bytecode(later_code), + ); + + let handler = snap_handler(provider); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetByteCodes(reth_eth_wire::snap::GetByteCodesMessage { + request_id: 3, + hashes: vec![KECCAK_EMPTY, B256::repeat_byte(0xff), code_hash, later_code_hash], + response_bytes: 1, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::ByteCodes(ByteCodesMessage { + request_id: 3, + codes: vec![Bytes::new(), code], + })) + ); + } + + #[tokio::test] + async fn snap_storage_ranges_limit_account_lookups() { + let provider = MockEthProvider::default(); + for _ in 0..=MAX_STORAGE_RANGE_ACCOUNTS_SERVE { + provider.push_snap_storage_range(Vec::new(), RangeEnd::Exhausted); + } + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 4, + root_hash: B256::ZERO, + account_hashes: vec![B256::ZERO; MAX_STORAGE_RANGE_ACCOUNTS_SERVE + 1], + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 4, + slots: vec![Vec::new(); MAX_STORAGE_RANGE_ACCOUNTS_SERVE], + proof: Vec::new(), + })) + ); + assert_eq!(provider.snap_storage_ranges_remaining(), 1); + assert_eq!(provider.snap_state_range_resolutions(), 1); + } + + #[tokio::test] + async fn snap_storage_range_proves_finite_limit_from_zero_origin() { + let provider = MockEthProvider::default(); + let hash = B256::repeat_byte(0x01); + let proof = vec![Bytes::from_static(&[0xcc])]; + // More entries exist beyond `limit_hash`, so the cursor stopped at the hash limit + // rather than exhausting the trie -- a proof is required even though origin is zero. + provider.push_snap_storage_range(vec![(hash, U256::from(1))], RangeEnd::HashLimit); + provider.set_snap_storage_proof(Some(proof.clone())); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 5, + root_hash: B256::ZERO, + account_hashes: vec![B256::repeat_byte(0x03)], + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0x20), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 5, + slots: vec![vec![StorageData { hash, data: Bytes::from_static(&[0x01]) }]], + proof, + })) + ); + } + + #[tokio::test] + async fn snap_storage_ranges_are_entirely_empty_when_an_account_is_missing() { + let provider = MockEthProvider::default(); + provider.push_missing_snap_storage_account(); + provider.push_snap_storage_range( + vec![(B256::repeat_byte(0x01), U256::from(1))], + RangeEnd::Exhausted, + ); + let missing_account = B256::repeat_byte(0x01); + let valid_account = B256::repeat_byte(0x02); + + let handler = snap_handler(provider.clone()); + let (response, rx) = oneshot::channel(); + handler.on_snap_request( + PeerId::default(), + SnapProtocolMessage::GetStorageRanges(GetStorageRangesMessage { + request_id: 6, + root_hash: B256::ZERO, + account_hashes: vec![missing_account, valid_account], + starting_hash: B256::ZERO, + limit_hash: B256::repeat_byte(0xff), + response_bytes: SOFT_RESPONSE_LIMIT as u64, + }), + response, + ); + + assert_eq!( + rx.await.unwrap(), + Ok(SnapResponse::StorageRanges(StorageRangesMessage { + request_id: 6, + slots: Vec::new(), + proof: Vec::new(), + })) + ); + // The valid account's queued range is never consumed: the response bails out at the + // first missing account instead of skipping it and shifting later positions. + assert_eq!(provider.snap_storage_ranges_remaining(), 1); + } } diff --git a/crates/net/network/src/test_utils/testnet.rs b/crates/net/network/src/test_utils/testnet.rs index b0983d84e5f..bc8a166e08f 100644 --- a/crates/net/network/src/test_utils/testnet.rs +++ b/crates/net/network/src/test_utils/testnet.rs @@ -29,7 +29,7 @@ use reth_network_api::{ use reth_network_peers::PeerId; use reth_storage_api::{ noop::NoopProvider, BalProvider, BlockReader, BlockReaderIdExt, HeaderProvider, - StateProviderFactory, + StateProviderFactory, StateRangeProviderFactory, }; use reth_tasks::Runtime; use reth_tokio_util::EventStream; @@ -249,6 +249,8 @@ where Header = alloy_consensus::Header, > + HeaderProvider + BalProvider + + StateProviderFactory + + StateRangeProviderFactory + Clone + Unpin + 'static, @@ -322,6 +324,8 @@ where Header = alloy_consensus::Header, > + HeaderProvider + BalProvider + + StateProviderFactory + + StateRangeProviderFactory + Unpin + 'static, Pool: TransactionPool< @@ -589,6 +593,8 @@ where Header = alloy_consensus::Header, > + HeaderProvider + BalProvider + + StateProviderFactory + + StateRangeProviderFactory + Unpin + 'static, Pool: TransactionPool< diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 68c79a720dc..1605bf416d2 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,7 +1,7 @@ use crate::{ providers::{ - ConsistentProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider, - StaticFileProviderRWRefMut, + ConsistentProvider, OverlayBuilder, OverlayStateProvider, OverlayStateProviderFactory, + ProviderNodeTypes, RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut, }, AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, @@ -11,9 +11,9 @@ use crate::{ RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; -use alloy_consensus::transaction::TransactionMeta; +use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; -use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; +use alloy_primitives::{Address, BlockHash, BlockNumber, Bytes, TxHash, TxNumber, B256}; use alloy_rpc_types_engine::ForkchoiceState; use reth_chain_state::{ BlockState, CanonicalInMemoryState, ForkChoiceNotifications, ForkChoiceSubscriptions, @@ -29,9 +29,19 @@ use reth_primitives_traits::{ use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; -use reth_storage_api::{BlockBodyIndicesProvider, NodePrimitivesProvider, StorageChangeSetReader}; +use reth_storage_api::{ + BlockBodyIndicesProvider, NodePrimitivesProvider, RangeEnd, RangeResponse, RangeResult, + StateRangeProvider, StateRangeProviderFactory, StateRangeView, StorageChangeSetReader, + StorageRangeResult, +}; use reth_storage_errors::provider::ProviderResult; -use reth_trie::{HashedPostState, KeccakKeyHasher}; +use reth_trie::{ + hashed_cursor::{HashedCursor, HashedCursorFactory}, + metrics::TrieRootMetrics, + proof::{Proof, StorageProof}, + HashedPostState, KeccakKeyHasher, MultiProofTargets, StorageRoot, TrieInput, TrieInputSorted, + TrieType, +}; use revm::database::BundleState; use std::{ ops::{RangeBounds, RangeInclusive}, @@ -40,6 +50,11 @@ use std::{ }; use tracing::trace; +const SNAPSHOT_STATE_RETENTION: u64 = 128; + +type StateRangeDbProvider = as DatabaseProviderFactory>::Provider; +type HistoricalStateRangeProvider = OverlayStateProvider>; + /// The main type for interacting with the blockchain. /// /// This type serves as the main entry point for interacting with the blockchain and provides data @@ -141,6 +156,72 @@ impl BlockchainProvider { let latest_historical = self.database.history_by_block_hash(anchor_hash)?; Ok(state.state_provider(latest_historical)) } + + /// Returns a cursor-backed state view for a state root still only in canonical in-memory + /// blocks, overlaying their merged trie state on the persisted anchor. + fn block_state_range_provider( + &self, + state_root: B256, + ) -> ProviderResult>> { + let Some(matched) = self + .canonical_in_memory_state + .canonical_chain() + .find(|state| state.state_root() == state_root) + else { + return Ok(None) + }; + + // Merge each in-memory block's trie delta, anchor to `matched`, oldest to newest. + let blocks: Vec<_> = matched.chain().map(|state| state.block()).collect(); + let sorted: Vec<_> = + blocks.iter().rev().map(|block| (block.hashed_state(), block.trie_updates())).collect(); + let input = TrieInput::from_blocks_sorted( + sorted.iter().map(|(state, nodes)| (state.as_ref(), nodes.as_ref())), + ); + let merged = TrieInputSorted::from_unsorted(input); + + // Anchor at the persisted block; the overlay reverts any db-tip advancement past it + // via changesets, then the merged in-memory delta applies on top. + let overlay_factory = OverlayStateProviderFactory::new( + self.database.clone(), + OverlayBuilder::::new( + matched.anchor().hash, + self.database.changeset_cache(), + ) + .with_hashed_state_overlay(Some(merged.state)) + .with_trie_updates_overlay(Some(merged.nodes)), + ); + reth_storage_api::DatabaseProviderROFactory::database_provider_ro(&overlay_factory) + .map(Some) + } + + /// Returns a cursor-backed state view for a retained canonical state root. + fn historical_state_range_provider( + &self, + state_root: B256, + ) -> ProviderResult>> { + let provider = self.database.provider()?; + let Some(finish) = provider.get_stage_checkpoint(StageId::Finish)? else { return Ok(None) }; + let oldest = finish.block_number.saturating_sub(SNAPSHOT_STATE_RETENTION - 1); + let mut block_hash = None; + + for number in (oldest..=finish.block_number).rev() { + let Some(header) = provider.sealed_header(number)? else { continue }; + if header.state_root() == state_root { + block_hash = Some(header.hash()); + break + } + } + drop(provider); + + let Some(block_hash) = block_hash else { return Ok(None) }; + let overlay_factory = OverlayStateProviderFactory::new( + self.database.clone(), + OverlayBuilder::::new(block_hash, self.database.changeset_cache()), + ); + reth_storage_api::DatabaseProviderROFactory::database_provider_ro(&overlay_factory) + .map(Some) + } } impl NodePrimitivesProvider for BlockchainProvider { @@ -153,17 +234,148 @@ impl BalProvider for BlockchainProvider { } } +/// State range view backed by one resolved historical overlay. +struct HistoricalStateRangeView { + provider: HistoricalStateRangeProvider, +} + +impl StateRangeProviderFactory for BlockchainProvider { + /// Resolves a retained canonical state root into a pinned range view, preferring a still + /// in-memory block over the persisted-history fallback. + fn state_range_provider(&self, state_root: B256) -> ProviderResult> { + let provider = match self.block_state_range_provider(state_root)? { + Some(provider) => Some(provider), + None => self.historical_state_range_provider(state_root)?, + }; + Ok(provider + .map(|provider| Box::new(HistoricalStateRangeView { provider }) as StateRangeView)) + } +} + +impl StateRangeProvider for HistoricalStateRangeView { + fn account_range( + &self, + start: B256, + limit: B256, + response_bytes: usize, + ) -> RangeResult<(B256, Account)> { + let mut cursor = self.provider.hashed_account_cursor().map_err(ProviderError::Database)?; + + let mut accounts = Vec::new(); + let mut total_bytes = 0usize; + let mut end = RangeEnd::Exhausted; + + // Append before checking `limit`, so an empty `[start, limit]` still returns the account + // right past `limit`, provable as an empty range rather than a skipped one. + let mut entry = cursor.seek(start).map_err(ProviderError::Database)?; + while let Some((hash, account)) = entry { + total_bytes += 32 + 4 * 32; // hash + rough upper bound of the RLP account body + accounts.push((hash, account)); + if hash >= limit { + end = RangeEnd::HashLimit; + break + } + if total_bytes > response_bytes { + end = RangeEnd::ByteLimit; + break + } + entry = cursor.next().map_err(ProviderError::Database)?; + } + + Ok(RangeResponse { items: accounts, end }) + } + + fn storage_root_by_hash(&self, hashed_address: B256) -> ProviderResult { + let root = StorageRoot::new_hashed( + &self.provider, + &self.provider, + hashed_address, + Default::default(), + TrieRootMetrics::new(TrieType::Storage), + ) + .root() + .map_err(|err| ProviderError::Database(err.into()))?; + Ok(root) + } + + fn storage_range( + &self, + hashed_address: B256, + start: B256, + limit: B256, + response_bytes: usize, + ) -> StorageRangeResult { + // Distinguish an absent account from one with no storage, so callers don't silently + // omit it and shift later accounts' positions. + let mut account_cursor = + self.provider.hashed_account_cursor().map_err(ProviderError::Database)?; + let found = account_cursor.seek(hashed_address).map_err(ProviderError::Database)?; + if found.map(|(hash, _)| hash) != Some(hashed_address) { + return Ok(None) + } + + let mut cursor = + self.provider.hashed_storage_cursor(hashed_address).map_err(ProviderError::Database)?; + + let mut slots = Vec::new(); + let mut total_bytes = 0usize; + let mut end = RangeEnd::Exhausted; + + // Append before checking `limit`, so an empty `[start, limit]` still returns the slot + // right past `limit`, provable as an empty range rather than a skipped one. + let mut entry = cursor.seek(start).map_err(ProviderError::Database)?; + while let Some((hash, value)) = entry { + total_bytes += 64; + slots.push((hash, value)); + if hash >= limit { + end = RangeEnd::HashLimit; + break + } + if total_bytes > response_bytes { + end = RangeEnd::ByteLimit; + break + } + entry = cursor.next().map_err(ProviderError::Database)?; + } + + Ok(Some(RangeResponse { items: slots, end })) + } + + fn account_range_proof(&self, keys: &[B256]) -> ProviderResult> { + let multiproof = Proof::new(&self.provider, &self.provider) + .multiproof(MultiProofTargets::accounts(keys.iter().copied())) + .map_err(ProviderError::from)?; + Ok(multiproof + .account_subtree + .into_nodes_sorted() + .into_iter() + .map(|(_, bytes)| bytes) + .collect()) + } + + fn storage_range_proof( + &self, + hashed_address: B256, + keys: &[B256], + ) -> ProviderResult> { + let multiproof = StorageProof::new_hashed(&self.provider, &self.provider, hashed_address) + .storage_multiproof(keys.iter().copied().collect()) + .map_err(ProviderError::from)?; + Ok(multiproof.subtree.into_nodes_sorted().into_iter().map(|(_, bytes)| bytes).collect()) + } +} + impl DatabaseProviderFactory for BlockchainProvider { type DB = N::DB; type Provider = as DatabaseProviderFactory>::Provider; type ProviderRW = as DatabaseProviderFactory>::ProviderRW; fn database_provider_ro(&self) -> ProviderResult { - self.database.database_provider_ro() + DatabaseProviderFactory::database_provider_ro(&self.database) } fn database_provider_rw(&self) -> ProviderResult { - self.database.database_provider_rw() + DatabaseProviderFactory::database_provider_rw(&self.database) } } @@ -796,6 +1008,7 @@ impl StateReader for BlockchainProvider { #[cfg(test)] mod tests { + use super::SNAPSHOT_STATE_RETENTION; use crate::{ providers::BlockchainProvider, test_utils::{ @@ -804,8 +1017,9 @@ mod tests { }, BlockWriter, CanonChainTracker, ProviderFactory, SaveBlocksMode, }; + use alloy_consensus::constants::EMPTY_ROOT_HASH; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; - use alloy_primitives::{BlockNumber, TxNumber, B256}; + use alloy_primitives::{keccak256, Address, BlockNumber, TxNumber, B256, U256}; use itertools::Itertools; use rand::Rng; use reth_chain_state::{ @@ -819,17 +1033,23 @@ mod tests { use reth_execution_types::{ BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome, }; - use reth_primitives_traits::{RecoveredBlock, SealedBlock, SignerRecoverable}; + use reth_primitives_traits::{ + Account, Block as _, RecoveredBlock, SealedBlock, SignerRecoverable, StorageEntry, + }; + use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_api::{ BlockBodyIndicesProvider, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, ChangeSetReader, DBProvider, DatabaseProviderFactory, - HeaderProvider, ReceiptProvider, ReceiptProviderIdExt, StateProviderFactory, - StateWriteConfig, StateWriter, TransactionVariant, TransactionsProvider, + HashingWriter, HeaderProvider, RangeEnd, ReceiptProvider, ReceiptProviderIdExt, + StageCheckpointWriter, StateProviderFactory, StateRangeProvider, StateRangeProviderFactory, + StateRootProvider, StateWriteConfig, StateWriter, StorageRootProvider, TransactionVariant, + TransactionsProvider, }; use reth_testing_utils::generators::{ self, random_block, random_block_range, random_changeset_range, random_eoa_accounts, random_receipt, BlockParams, BlockRangeParams, }; + use reth_trie::{updates::TrieUpdates, ComputedTrieData, HashedPostState, HashedStorage}; use revm::database::{BundleState, OriginalValuesKnown}; use std::{ collections::BTreeMap, @@ -2616,4 +2836,344 @@ mod tests { Ok(()) } + + fn random_account(nonce: u64) -> (Address, Account) { + (Address::random(), Account { nonce, balance: U256::from(nonce), bytecode_hash: None }) + } + + /// [`BlockchainProvider::new`] needs a genesis header to initialize its chain tracker. + fn test_provider_factory_with_genesis() -> eyre::Result> { + let factory = create_test_provider_factory(); + let provider_rw = factory.provider_rw()?; + let mut rng = generators::rng(); + let genesis = + random_block(&mut rng, 0, BlockParams { tx_count: Some(0), ..Default::default() }); + provider_rw + .insert_block(&genesis.try_recover().expect("failed to seal block with senders"))?; + provider_rw.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(0))?; + provider_rw.commit()?; + Ok(factory) + } + + #[test] + fn state_range_provider_account_range_is_sorted_and_bounded() -> eyre::Result<()> { + let factory = test_provider_factory_with_genesis()?; + let provider_rw = factory.provider_rw()?; + + let accounts: Vec<_> = (0..5u64).map(random_account).collect(); + provider_rw.insert_account_for_hashing( + accounts.iter().map(|(address, account)| (*address, Some(*account))), + )?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + + let mut expected: Vec<_> = + accounts.iter().map(|(address, account)| (keccak256(address), *account)).collect(); + expected.sort_by_key(|(hash, _)| *hash); + let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); + + let all = state.account_range(B256::ZERO, B256::repeat_byte(0xff), 10_000)?; + assert_eq!(all.end, RangeEnd::Exhausted); + assert_eq!(all.items, expected); + + // The limit exactly matches the second account's hash, so the range ends there rather + // than by exhausting the trie. + let bounded = state.account_range(B256::ZERO, expected[1].0, 10_000)?; + assert_eq!(bounded.end, RangeEnd::HashLimit); + assert_eq!(bounded.items, expected[..2]); + + Ok(()) + } + + #[test] + fn state_range_provider_account_range_respects_response_bytes() -> eyre::Result<()> { + let factory = test_provider_factory_with_genesis()?; + let provider_rw = factory.provider_rw()?; + + let accounts: Vec<_> = (0..5u64).map(random_account).collect(); + provider_rw.insert_account_for_hashing( + accounts.iter().map(|(address, account)| (*address, Some(*account))), + )?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); + + // Budget only fits a single account. + let partial = state.account_range(B256::ZERO, B256::repeat_byte(0xff), 150)?; + assert_eq!(partial.end, RangeEnd::ByteLimit); + assert_eq!(partial.items.len(), 1); + + Ok(()) + } + + #[test] + fn state_range_provider_storage_range_and_root() -> eyre::Result<()> { + let factory = test_provider_factory_with_genesis()?; + let provider_rw = factory.provider_rw()?; + + let (address, account) = random_account(1); + let hashed_address = keccak256(address); + provider_rw.insert_account_for_hashing([(address, Some(account))])?; + let slots = [ + StorageEntry { key: B256::with_last_byte(1), value: U256::from(10) }, + StorageEntry { key: B256::with_last_byte(2), value: U256::from(20) }, + ]; + provider_rw.insert_storage_for_hashing([(address, slots)])?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); + + let expected_root = provider.latest()?.storage_root(address, HashedStorage::default())?; + assert_eq!(state.storage_root_by_hash(hashed_address)?, expected_root); + + let returned = state + .storage_range(hashed_address, B256::ZERO, B256::repeat_byte(0xff), 10_000)? + .unwrap(); + assert_eq!(returned.end, RangeEnd::Exhausted); + let mut expected: Vec<_> = + slots.iter().map(|entry| (keccak256(entry.key), entry.value)).collect(); + expected.sort_by_key(|(hash, _)| *hash); + assert_eq!(returned.items, expected); + + // `start == limit == ZERO` means the first real slot's hash already reaches the limit. + let empty_window = + state.storage_range(hashed_address, B256::ZERO, B256::ZERO, 10_000)?.unwrap(); + assert_eq!(empty_window.end, RangeEnd::HashLimit); + assert_eq!(empty_window.items, expected[..1]); + + // An account absent from the trie is distinguished from one with no storage. + assert!(state + .storage_range(B256::repeat_byte(0xee), B256::ZERO, B256::repeat_byte(0xff), 10_000)? + .is_none()); + + Ok(()) + } + + #[test] + fn state_range_provider_proofs_start_at_the_real_root() -> eyre::Result<()> { + let factory = test_provider_factory_with_genesis()?; + let provider_rw = factory.provider_rw()?; + + let (address, account) = random_account(1); + let hashed_address = keccak256(address); + let hashed_slot = keccak256(B256::with_last_byte(1)); + provider_rw.insert_account_for_hashing([(address, Some(account))])?; + provider_rw.insert_storage_for_hashing([( + address, + [StorageEntry { key: B256::with_last_byte(1), value: U256::from(10) }], + )])?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + + // The first node of a sorted boundary proof is always the trie root, so this checks the + // proof was generated against the real, current root rather than a stale or empty one. + let state_root = provider.latest()?.state_root(HashedPostState::default())?; + let state = provider.state_range_provider(EMPTY_ROOT_HASH)?.unwrap(); + let account_proof = state.account_range_proof(&[hashed_address])?; + assert!(!account_proof.is_empty()); + assert_eq!(keccak256(&account_proof[0]), state_root); + + let storage_root = state.storage_root_by_hash(hashed_address)?; + let storage_proof = state.storage_range_proof(hashed_address, &[hashed_slot])?; + assert!(!storage_proof.is_empty()); + assert_eq!(keccak256(&storage_proof[0]), storage_root); + + Ok(()) + } + + #[test] + fn state_range_provider_serves_recent_root_and_rejects_expired_root() -> eyre::Result<()> { + let mut rng = generators::rng(); + let factory = create_test_provider_factory(); + let provider_rw = factory.provider_rw()?; + let expired_root = B256::repeat_byte(0x11); + let recent_root = B256::repeat_byte(0x22); + let mut parent = B256::ZERO; + + for number in 0..=SNAPSHOT_STATE_RETENTION { + let mut block = random_block( + &mut rng, + number, + BlockParams { parent: Some(parent), tx_count: Some(0), ..Default::default() }, + ) + .unseal(); + block.header.state_root = match number { + 0 => expired_root, + 64 => recent_root, + _ => EMPTY_ROOT_HASH, + }; + let block = block.seal_slow(); + parent = block.hash(); + provider_rw + .insert_block(&block.try_recover().expect("failed to seal block with senders"))?; + } + provider_rw.save_stage_checkpoint( + StageId::Finish, + StageCheckpoint::new(SNAPSHOT_STATE_RETENTION), + )?; + provider_rw.commit()?; + + let provider = BlockchainProvider::new(factory)?; + assert!(provider.state_range_provider(recent_root)?.is_some()); + assert!(provider.state_range_provider(expired_root)?.is_none()); + + Ok(()) + } + + #[test] + fn state_range_provider_serves_persisted_root_with_in_memory_overlay() -> eyre::Result<()> { + let mut rng = generators::rng(); + let (provider, _, _, _) = provider_with_random_blocks( + &mut rng, + TEST_BLOCKS_COUNT - 1, + 1, + BlockRangeParams::default(), + )?; + assert!(provider.canonical_in_memory_state.head_state().is_some()); + let provider_rw = provider.database.provider_rw()?; + provider_rw.save_stage_checkpoint( + StageId::Finish, + StageCheckpoint::new((TEST_BLOCKS_COUNT - 2) as u64), + )?; + provider_rw.commit()?; + + assert!(provider.state_range_provider(EMPTY_ROOT_HASH)?.is_some()); + + Ok(()) + } + + #[test] + fn state_range_provider_resolves_root_from_in_memory_block() -> eyre::Result<()> { + let mut rng = generators::rng(); + let factory = test_provider_factory_with_genesis()?; + let provider = BlockchainProvider::new(factory)?; + + let (address, account) = random_account(1); + let hashed_address = keccak256(address); + let mut hashed_state = HashedPostState::default(); + hashed_state.accounts.insert(hashed_address, Some(account)); + + // A root only the in-memory block carries, so a match proves the in-memory path (not + // persisted history, which has no block with this root) resolved it. + let unique_root = B256::repeat_byte(0x77); + let parent = provider.canonical_in_memory_state.get_canonical_head(); + let mut block = random_block( + &mut rng, + parent.number + 1, + BlockParams { parent: Some(parent.hash()), tx_count: Some(0), ..Default::default() }, + ) + .unseal(); + block.header.state_root = unique_root; + let block = block.seal_slow().try_recover().expect("failed to seal block with senders"); + + let trie_data = ComputedTrieData::new( + Arc::new(hashed_state.into_sorted()), + Arc::new(TrieUpdates::default().into_sorted()), + ); + let execution_output = BlockExecutionOutput { + result: BlockExecutionResult { + receipts: Default::default(), + requests: Default::default(), + gas_used: 0, + blob_gas_used: 0, + }, + state: Default::default(), + }; + let executed = ExecutedBlock::new(Arc::new(block), Arc::new(execution_output), trie_data); + provider + .canonical_in_memory_state + .update_chain(NewCanonicalChain::Commit { new: vec![executed] }); + + let state = + provider.state_range_provider(unique_root)?.expect("in-memory root must resolve"); + let range = state.account_range(B256::ZERO, B256::repeat_byte(0xff), 10_000)?; + assert_eq!(range.items, vec![(hashed_address, account)]); + + Ok(()) + } + + #[test] + fn state_range_provider_reverts_database_advancement_past_anchor() -> eyre::Result<()> { + let mut rng = generators::rng(); + let factory = test_provider_factory_with_genesis()?; + let provider = BlockchainProvider::new(factory)?; + let genesis = provider.canonical_in_memory_state.get_canonical_head(); + + // In-memory target block anchored on genesis, with a known account. + let (target_address, target_account) = random_account(1); + let target_hashed = keccak256(target_address); + let mut target_state = HashedPostState::default(); + target_state.accounts.insert(target_hashed, Some(target_account)); + + let unique_root = B256::repeat_byte(0x77); + let mut block = random_block( + &mut rng, + genesis.number + 1, + BlockParams { parent: Some(genesis.hash()), tx_count: Some(0), ..Default::default() }, + ) + .unseal(); + block.header.state_root = unique_root; + let block = block.seal_slow().try_recover().expect("failed to seal block with senders"); + let trie_data = ComputedTrieData::new( + Arc::new(target_state.into_sorted()), + Arc::new(TrieUpdates::default().into_sorted()), + ); + let execution_output = BlockExecutionOutput { + result: BlockExecutionResult { + receipts: Default::default(), + requests: Default::default(), + gas_used: 0, + blob_gas_used: 0, + }, + state: Default::default(), + }; + let executed = ExecutedBlock::new(Arc::new(block), Arc::new(execution_output), trie_data); + provider + .canonical_in_memory_state + .update_chain(NewCanonicalChain::Commit { new: vec![executed] }); + + // Persistence races ahead: a *different* block, with a *different* account, lands in + // the database on top of the same genesis anchor while the in-memory chain above still + // references genesis as its anchor. + let (noise_address, noise_account) = random_account(2); + let noise_block = random_block( + &mut rng, + genesis.number + 1, + BlockParams { parent: Some(genesis.hash()), tx_count: Some(0), ..Default::default() }, + ) + .try_recover() + .expect("failed to seal block with senders"); + let mut noise_state = HashedPostState::default(); + noise_state.accounts.insert(keccak256(noise_address), Some(noise_account)); + let provider_rw = provider.database.provider_rw()?; + provider_rw.append_blocks_with_state( + vec![noise_block], + &ExecutionOutcome { + bundle: BundleState::new( + [(noise_address, None, Some(noise_account.into()), Default::default())], + [[(noise_address, Some(None), [])]], + [], + ), + first_block: genesis.number + 1, + ..Default::default() + }, + noise_state.into_sorted(), + )?; + provider_rw + .save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(genesis.number + 1))?; + provider_rw.commit()?; + + // Resolving the in-memory root must revert the database's advancement back to genesis, + // so the noise account must not leak into the result. + let state = + provider.state_range_provider(unique_root)?.expect("in-memory root must resolve"); + let range = state.account_range(B256::ZERO, B256::repeat_byte(0xff), 10_000)?; + assert_eq!(range.items, vec![(target_hashed, target_account)]); + + Ok(()) + } } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 40ec3834865..9f6785774af 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -204,6 +204,11 @@ impl ProviderFactory { self } + /// Returns the shared changeset cache. + pub(crate) fn changeset_cache(&self) -> ChangesetCache { + self.changeset_cache.clone() + } + /// Sets the minimum pruning distance for an existing [`ProviderFactory`]. /// /// This controls the minimum distance from tip required before pruning can occur. diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index 6d7b88db881..f8894ca2589 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -159,15 +159,38 @@ impl OverlayBuilder { mut self, hashed_state_overlay: Option>, ) -> Self { - if let Some(state) = hashed_state_overlay { + if let Some(new_state) = hashed_state_overlay { match &mut self.overlay_source { - Some(OverlaySource::Managed { state: managed_state, .. }) => { - *managed_state = state; - } - _ => { + Some( + OverlaySource::Managed { state, .. } | OverlaySource::Immediate { state, .. }, + ) => *state = new_state, + None => { self.overlay_source = Some(OverlaySource::Immediate { trie: Arc::new(TrieUpdatesSorted::default()), - state, + state: new_state, + }); + } + } + } + self + } + + /// Set the trie updates overlay. + /// + /// Only applies to an immediate overlay: a managed overlay's trie updates are resolved from + /// its manager instead. + pub fn with_trie_updates_overlay( + mut self, + trie_updates_overlay: Option>, + ) -> Self { + if let Some(trie) = trie_updates_overlay { + match &mut self.overlay_source { + Some(OverlaySource::Immediate { trie: existing, .. }) => *existing = trie, + Some(OverlaySource::Managed { .. }) => {} + None => { + self.overlay_source = Some(OverlaySource::Immediate { + trie, + state: Arc::new(HashedPostStateSorted::default()), }); } } diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 59b90e02643..68244202e1d 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -2,8 +2,10 @@ use crate::{ traits::{BlockSource, ReceiptProvider}, AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, HeaderProvider, - PruneCheckpointReader, ReceiptProviderIdExt, StateProvider, StateProviderBox, - StateProviderFactory, StateReader, StateRootProvider, TransactionVariant, TransactionsProvider, + PruneCheckpointReader, RangeEnd, RangeResponse, RangeResult, ReceiptProviderIdExt, + StateProvider, StateProviderBox, StateProviderFactory, StateRangeProvider, + StateRangeProviderFactory, StateRangeView, StateReader, StateRootProvider, StorageRangeResult, + TransactionVariant, TransactionsProvider, }; use alloy_consensus::{ constants::EMPTY_ROOT_HASH, @@ -43,10 +45,13 @@ use reth_trie::{ MultiProofTargets, StorageMultiProof, StorageProof, TrieInput, }; use std::{ - collections::BTreeMap, + collections::{BTreeMap, VecDeque}, fmt::Debug, ops::{RangeBounds, RangeInclusive}, - sync::Arc, + sync::{ + atomic::{AtomicBool, AtomicUsize, Ordering}, + Arc, + }, }; use tokio::sync::broadcast; @@ -70,10 +75,43 @@ pub struct MockEthProvider>>, /// Local BAL store handle pub bal_store: BalStoreHandle, + /// Whether snap state reads should fail for handler error-path tests. + snap_state_reads_fail: Arc, + /// Whether a snap state range view is available. + snap_state_range_available: Arc, + /// Number of snap state range view resolutions. + snap_state_range_resolutions: Arc, + /// Account range returned to snap handler tests. + snap_account_range: Arc>, + /// Storage roots returned to snap handler tests, keyed by hashed address. + snap_storage_roots: Arc>>, + /// Storage ranges returned to snap handler tests. + snap_storage_ranges: Arc>>, + /// Storage range requests observed by snap handler tests. + snap_storage_range_requests: Arc>>, + /// Account proof returned to snap handler tests. + snap_account_proof: Arc>>>, + /// Storage proof returned to snap handler tests. + snap_storage_proof: Arc>>>, tx: TxMock, prune_modes: Arc, } +/// Optional mock account entries paired with why the range ended. +type MockAccountRange = Option<(Vec<(B256, Account)>, RangeEnd)>; +/// Outcome of a queued mock `storage_range` call. +#[derive(Debug, Clone)] +enum MockStorageRangeOutcome { + /// The provider fails this call (e.g. simulating a database error). + Error, + /// The requested account isn't present in the pinned state. + AccountMissing, + /// The account is present; these are its slots and why the range ended. + Found(Vec<(B256, U256)>, RangeEnd), +} +/// Hashed address, origin, limit, and byte budget of a mock storage range request. +type MockStorageRangeRequest = (B256, B256, B256, usize); + impl Clone for MockEthProvider where T::Block: Clone, @@ -88,6 +126,15 @@ where state_roots: self.state_roots.clone(), block_body_indices: self.block_body_indices.clone(), bal_store: self.bal_store.clone(), + snap_state_reads_fail: self.snap_state_reads_fail.clone(), + snap_state_range_available: self.snap_state_range_available.clone(), + snap_state_range_resolutions: self.snap_state_range_resolutions.clone(), + snap_account_range: self.snap_account_range.clone(), + snap_storage_roots: self.snap_storage_roots.clone(), + snap_storage_ranges: self.snap_storage_ranges.clone(), + snap_storage_range_requests: self.snap_storage_range_requests.clone(), + snap_account_proof: self.snap_account_proof.clone(), + snap_storage_proof: self.snap_storage_proof.clone(), tx: self.tx.clone(), prune_modes: self.prune_modes.clone(), } @@ -106,6 +153,15 @@ impl MockEthProvider { state_roots: Default::default(), block_body_indices: Default::default(), bal_store: Default::default(), + snap_state_reads_fail: Default::default(), + snap_state_range_available: Default::default(), + snap_state_range_resolutions: Default::default(), + snap_account_range: Default::default(), + snap_storage_roots: Default::default(), + snap_storage_ranges: Default::default(), + snap_storage_range_requests: Default::default(), + snap_account_proof: Default::default(), + snap_storage_proof: Default::default(), tx: Default::default(), prune_modes: Default::default(), } @@ -113,6 +169,72 @@ impl MockEthProvider { } impl MockEthProvider { + /// Makes snap state reads return provider errors when `fail` is true. + pub fn set_snap_state_reads_fail(&self, fail: bool) { + self.snap_state_reads_fail.store(fail, Ordering::Relaxed); + } + + /// Sets the available account range returned to snap handler tests. + pub fn set_snap_account_range(&self, accounts: Vec<(B256, Account)>, end: RangeEnd) { + self.snap_state_range_available.store(true, Ordering::Relaxed); + *self.snap_account_range.lock() = Some((accounts, end)); + } + + /// Sets an account's storage root for snap handler tests. + pub fn set_snap_storage_root(&self, hashed_address: B256, storage_root: B256) { + self.snap_storage_roots.lock().insert(hashed_address, storage_root); + } + + /// Adds an available storage range for the next snap handler call. + pub fn push_snap_storage_range(&self, slots: Vec<(B256, U256)>, end: RangeEnd) { + self.snap_state_range_available.store(true, Ordering::Relaxed); + self.snap_storage_ranges.lock().push_back(MockStorageRangeOutcome::Found(slots, end)); + } + + /// Marks the account for the next snap handler call as absent from the pinned state. + pub fn push_missing_snap_storage_account(&self) { + self.snap_state_range_available.store(true, Ordering::Relaxed); + self.snap_storage_ranges.lock().push_back(MockStorageRangeOutcome::AccountMissing); + } + + /// Adds an unavailable storage range for the next snap handler call. + pub fn push_unavailable_snap_storage_range(&self) { + self.snap_state_range_available.store(true, Ordering::Relaxed); + self.snap_storage_ranges.lock().push_back(MockStorageRangeOutcome::Error); + } + + /// Returns the number of queued storage ranges for snap handler tests. + pub fn snap_storage_ranges_remaining(&self) -> usize { + self.snap_storage_ranges.lock().len() + } + + /// Returns the storage range requests observed by snap handler tests. + pub fn snap_storage_range_requests(&self) -> Vec<(B256, B256, B256, usize)> { + self.snap_storage_range_requests.lock().clone() + } + + /// Returns the number of snap state range view resolutions. + pub fn snap_state_range_resolutions(&self) -> usize { + self.snap_state_range_resolutions.load(Ordering::Relaxed) + } + + /// Sets the account proof returned to snap handler tests. + pub fn set_snap_account_proof(&self, proof: Option>) { + *self.snap_account_proof.lock() = proof; + } + + /// Sets the storage proof returned to snap handler tests. + pub fn set_snap_storage_proof(&self, proof: Option>) { + *self.snap_storage_proof.lock() = proof; + } + + fn ensure_snap_state_reads_succeed(&self) -> ProviderResult<()> { + if self.snap_state_reads_fail.load(Ordering::Relaxed) { + return Err(ProviderError::BestBlockNotFound) + } + Ok(()) + } + /// Add block to local block store pub fn add_block(&self, hash: B256, block: T::Block) { self.add_header(hash, block.header().clone()); @@ -190,6 +312,15 @@ impl MockEthProvider { state_roots: self.state_roots, block_body_indices: self.block_body_indices, bal_store: self.bal_store, + snap_state_reads_fail: self.snap_state_reads_fail, + snap_state_range_available: self.snap_state_range_available, + snap_state_range_resolutions: self.snap_state_range_resolutions, + snap_account_range: self.snap_account_range, + snap_storage_roots: self.snap_storage_roots, + snap_storage_ranges: self.snap_storage_ranges, + snap_storage_range_requests: self.snap_storage_range_requests, + snap_account_proof: self.snap_account_proof, + snap_storage_proof: self.snap_storage_proof, tx: self.tx, prune_modes: self.prune_modes, } @@ -223,6 +354,82 @@ impl BalProvider for MockEthProvider } } +impl StateRangeProviderFactory for MockEthProvider +where + T: NodePrimitives, + T::Block: Clone, + ChainSpec: Send + Sync + 'static, +{ + fn state_range_provider(&self, _state_root: B256) -> ProviderResult> { + self.snap_state_range_resolutions.fetch_add(1, Ordering::Relaxed); + self.ensure_snap_state_reads_succeed()?; + if !self.snap_state_range_available.load(Ordering::Relaxed) { + return Ok(None) + } + Ok(Some(Box::new(self.clone()))) + } +} + +impl StateRangeProvider for MockEthProvider { + fn account_range( + &self, + _start: B256, + _limit: B256, + _response_bytes: usize, + ) -> RangeResult<(B256, Account)> { + self.ensure_snap_state_reads_succeed()?; + let (items, end) = + self.snap_account_range.lock().clone().ok_or(ProviderError::BestBlockNotFound)?; + Ok(RangeResponse { items, end }) + } + + fn storage_root_by_hash(&self, hashed_address: B256) -> ProviderResult { + self.ensure_snap_state_reads_succeed()?; + self.snap_storage_roots + .lock() + .get(&hashed_address) + .copied() + .ok_or(ProviderError::BestBlockNotFound) + } + + fn storage_range( + &self, + hashed_address: B256, + start: B256, + limit: B256, + response_bytes: usize, + ) -> StorageRangeResult { + self.ensure_snap_state_reads_succeed()?; + self.snap_storage_range_requests.lock().push(( + hashed_address, + start, + limit, + response_bytes, + )); + let outcome = + self.snap_storage_ranges.lock().pop_front().ok_or(ProviderError::BestBlockNotFound)?; + match outcome { + MockStorageRangeOutcome::Error => Err(ProviderError::BestBlockNotFound), + MockStorageRangeOutcome::AccountMissing => Ok(None), + MockStorageRangeOutcome::Found(items, end) => Ok(Some(RangeResponse { items, end })), + } + } + + fn account_range_proof(&self, _keys: &[B256]) -> ProviderResult> { + self.ensure_snap_state_reads_succeed()?; + self.snap_account_proof.lock().clone().ok_or(ProviderError::BestBlockNotFound) + } + + fn storage_range_proof( + &self, + _hashed_address: B256, + _keys: &[B256], + ) -> ProviderResult> { + self.ensure_snap_state_reads_succeed()?; + self.snap_storage_proof.lock().clone().ok_or(ProviderError::BestBlockNotFound) + } +} + /// An extended account for local store #[derive(Debug, Clone)] pub struct ExtendedAccount { @@ -933,6 +1140,7 @@ impl StatePr for MockEthProvider { fn latest(&self) -> ProviderResult { + self.ensure_snap_state_reads_succeed()?; Ok(Box::new(self.clone())) } diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index 070d8bfefd5..d3e6f36be45 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -3,8 +3,8 @@ use crate::{ AccountReader, BalProvider, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider, PruneCheckpointReader, - RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, StateReader, - StaticFileProviderFactory, + RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, StateRangeProviderFactory, + StateReader, StaticFileProviderFactory, }; use reth_chain_state::{ CanonStateSubscriptions, ForkChoiceSubscriptions, PersistedBlockSubscriptions, @@ -34,6 +34,7 @@ pub trait FullProvider: > + AccountReader + BalProvider + StateProviderFactory + + StateRangeProviderFactory + StateReader + HashedPostStateProvider + ChainSpecProvider @@ -71,6 +72,7 @@ impl FullProvider for T where > + AccountReader + BalProvider + StateProviderFactory + + StateRangeProviderFactory + StateReader + HashedPostStateProvider + ChainSpecProvider diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 80726e17c27..49a81d4ff26 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -7,8 +7,9 @@ use crate::{ BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, BytecodeReader, ChangeSetReader, HashedPostStateProvider, HeaderProvider, NodePrimitivesProvider, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, - StateProofProvider, StateProvider, StateProviderBox, StateProviderFactory, StateReader, - StateRootProvider, StorageRootProvider, TransactionVariant, TransactionsProvider, + StateProofProvider, StateProvider, StateProviderBox, StateProviderFactory, + StateRangeProviderFactory, StateRangeView, StateReader, StateRootProvider, StorageRootProvider, + TransactionVariant, TransactionsProvider, }; #[cfg(feature = "db-api")] @@ -117,6 +118,12 @@ impl BalProvider for NoopProvider { } } +impl StateRangeProviderFactory for NoopProvider { + fn state_range_provider(&self, _state_root: B256) -> ProviderResult> { + Ok(None) + } +} + /// Noop implementation for testing purposes impl BlockHashReader for NoopProvider { fn block_hash(&self, _number: u64) -> ProviderResult> { diff --git a/crates/storage/storage-api/src/trie.rs b/crates/storage/storage-api/src/trie.rs index b5c7de7b652..dc30bbf52ad 100644 --- a/crates/storage/storage-api/src/trie.rs +++ b/crates/storage/storage-api/src/trie.rs @@ -1,5 +1,6 @@ -use alloc::vec::Vec; -use alloy_primitives::{Address, Bytes, B256}; +use alloc::{boxed::Box, vec::Vec}; +use alloy_primitives::{Address, Bytes, B256, U256}; +use reth_primitives_traits::Account; use reth_storage_errors::provider::ProviderResult; use reth_trie_common::{ updates::{StorageTrieUpdatesSorted, TrieUpdates, TrieUpdatesSorted}, @@ -65,6 +66,82 @@ pub trait StorageRootProvider { ) -> ProviderResult; } +/// A type that can iterate over consecutive hashed accounts and storage slots, and generate +/// boundary proofs for them, for serving `snap/2` (EIP-8189) `GetAccountRange`/`GetStorageRanges` +/// requests. Hash-native throughout, unlike [`StorageRootProvider`]. +#[auto_impl::auto_impl(&, Box, Arc)] +pub trait StateRangeProvider { + /// Returns accounts (hash, account) in `[start, limit]`, bounded by `response_bytes`. + fn account_range( + &self, + start: B256, + limit: B256, + response_bytes: usize, + ) -> RangeResult<(B256, Account)>; + + /// Returns the storage root for `hashed_address` without needing its address preimage. + fn storage_root_by_hash(&self, hashed_address: B256) -> ProviderResult; + + /// Same as [`Self::account_range`], but for the storage slots of `hashed_address`. + /// + /// Returns `None` if `hashed_address` isn't present in the account trie at the pinned state + /// root, distinct from an account that is present but has no storage. + fn storage_range( + &self, + hashed_address: B256, + start: B256, + limit: B256, + response_bytes: usize, + ) -> StorageRangeResult; + + /// Returns an account-trie boundary proof for the already-hashed `keys`. + fn account_range_proof(&self, keys: &[B256]) -> ProviderResult>; + + /// Same as [`Self::account_range_proof`], but for the storage trie of `hashed_address`. + fn storage_range_proof( + &self, + hashed_address: B256, + keys: &[B256], + ) -> ProviderResult>; +} + +/// A type that resolves retained state roots into reusable state range views. +#[auto_impl::auto_impl(&, Arc)] +pub trait StateRangeProviderFactory { + /// Returns a view pinned to `state_root`, or `None` if that root is not retained. + fn state_range_provider(&self, state_root: B256) -> ProviderResult>; +} + +/// A range query's items and why the range ended where it did. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RangeResponse { + /// The items found within the requested range, in ascending key order. + pub items: Vec, + /// Why `items` doesn't necessarily continue past its last entry. + pub end: RangeEnd, +} + +/// Why a range query stopped before the caller-requested `limit`. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RangeEnd { + /// The cursor ran out of entries: `items` covers everything from `start` onward. + Exhausted, + /// The last returned item's key reached or passed the requested `limit`. + HashLimit, + /// `response_bytes` was exceeded before `limit` was reached. + ByteLimit, +} + +/// Result of a [`StateRangeProvider`] range query. +pub type RangeResult = ProviderResult>; + +/// Result of a [`StateRangeProvider::storage_range`] query: `None` if the account itself isn't +/// present in the trie at the pinned state root. +pub type StorageRangeResult = ProviderResult>>; + +/// A reusable state range view resolved for a specific state root. +pub type StateRangeView = Box; + /// A type that can generate state proof on top of a given post state. #[auto_impl::auto_impl(&, Box, Arc)] pub trait StateProofProvider { From 433a7bc3133379e2532128f7fed3ed698d3cfbd8 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Thu, 16 Jul 2026 20:27:56 +0530 Subject: [PATCH 008/148] fix(rpc): fail invalid testing block transactions by default (#26372) Co-authored-by: Matthias Seitz --- .github/scripts/bench-txgen-run.sh | 4 ++++ crates/node/core/src/args/rpc_server.rs | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh index 98ea31f04c3..5631ff70204 100755 --- a/.github/scripts/bench-txgen-run.sh +++ b/.github/scripts/bench-txgen-run.sh @@ -259,6 +259,10 @@ RETH_ARGS=( --no-persist-peers ) +if [ -n "${BENCH_REORG:-}" ]; then + RETH_ARGS+=(--testing.skip-invalid-transactions) +fi + SYNC_STATE_IDLE=false if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then RETH_ARGS+=(--debug.startup-sync-state-idle) diff --git a/crates/node/core/src/args/rpc_server.rs b/crates/node/core/src/args/rpc_server.rs index 5a55aeb1c4b..e04df70db74 100644 --- a/crates/node/core/src/args/rpc_server.rs +++ b/crates/node/core/src/args/rpc_server.rs @@ -673,7 +673,7 @@ pub struct RpcServerArgs { /// /// When enabled, transactions that fail execution will be skipped, and all subsequent /// transactions from the same sender will also be skipped. - #[arg(long = "testing.skip-invalid-transactions", default_value_t = true)] + #[arg(long = "testing.skip-invalid-transactions", default_value_t = false)] pub testing_skip_invalid_transactions: bool, /// Override the gas limit used by `testing_buildBlockV1`. @@ -925,7 +925,7 @@ impl Default for RpcServerArgs { rpc_state_cache, gas_price_oracle, rpc_send_raw_transaction_sync_timeout, - testing_skip_invalid_transactions: true, + testing_skip_invalid_transactions: false, testing_gas_limit: None, rpc_force_blob_sidecar_upcasting: false, } From 8ae521bdb6eac7297cc686596456b5526f99b7fe Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Thu, 16 Jul 2026 18:08:46 +0300 Subject: [PATCH 009/148] chore(deps): bump alloy core 1.6.1 (#26419) --- Cargo.lock | 85 ++++++++++++++++++++++++++++++++++-------------------- Cargo.toml | 6 ++-- 2 files changed, 56 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index bc0144c7b1c..106b84b57b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -187,9 +187,9 @@ dependencies = [ [[package]] name = "alloy-dyn-abi" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a475bb02d9cef2dbb99065c1664ab3fe1f9352e21d6d5ed3f02cdbfc06ed1abc" +checksum = "9a04eb4abc2b5074a18e687ee63918f407cc7990083cba9b999445f839796060" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -353,9 +353,9 @@ dependencies = [ [[package]] name = "alloy-json-abi" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7c36c9d7f9021601b04bfef14a4b64849f6d73116a4e91e071d7fbfe10247901" +checksum = "6cee30dd4c2f4b23f434fdf675e7bf9681b86768141277266c6f548ef25cba0a" dependencies = [ "alloy-primitives", "alloy-sol-type-parser", @@ -441,9 +441,9 @@ dependencies = [ [[package]] name = "alloy-primitives" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4885c1409b6936c4898e646ef58baf6ec54edaf6d8179f79df805a7b85b7cf3e" +checksum = "f007e257069855bdf21d27762fd3f3705a613f805c9a08309bf353503f081d71" dependencies = [ "alloy-rlp", "arbitrary", @@ -706,7 +706,7 @@ dependencies = [ "alloy-serde", "alloy-sol-types", "arbitrary", - "itertools 0.13.0", + "itertools 0.14.0", "serde", "serde_json", "serde_with", @@ -802,13 +802,13 @@ dependencies = [ [[package]] name = "alloy-sol-macro" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840128ed2b2971d6d4668a553fe403a82683d3acc646c73e75887e7157408033" +checksum = "b5655c38d5f84955bf727b2eeb62fddd91ebb98fd1d7ae6eb77f73ea88f9b9cf" dependencies = [ "alloy-sol-macro-expander", "alloy-sol-macro-input", - "proc-macro-error2", + "proc-macro-error3", "proc-macro2", "quote", "syn 2.0.118", @@ -816,15 +816,15 @@ dependencies = [ [[package]] name = "alloy-sol-macro-expander" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63ec265e5d65d725175f6ca7711c970824c90ef9c0d1f1973711d4150ee612dd" +checksum = "6277c780e07b76951e09a59788dde230d1582612324177d11a43a61e21a6bb83" dependencies = [ "alloy-sol-macro-input", "const-hex", "heck", "indexmap 2.14.0", - "proc-macro-error2", + "proc-macro-error3", "proc-macro2", "quote", "sha3 0.11.0", @@ -834,9 +834,9 @@ dependencies = [ [[package]] name = "alloy-sol-macro-input" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89bf01077f18650876cfa682eb1f949967b5cde03f1a51c955c469d2c9b4aa67" +checksum = "9762b2ad3e5a0c09886de54fe549ab0056681df843cb082e2df7e1c0eb270d30" dependencies = [ "const-hex", "dunce", @@ -850,9 +850,9 @@ dependencies = [ [[package]] name = "alloy-sol-type-parser" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "857b470ecdd2ed38beaf82ad1a38c516a8ff75266750f38b9eeed001d575241b" +checksum = "da4c7130f0f01f4719678bda3db3bc7267fc2f7f9d0565e3bd964cd2bb45050d" dependencies = [ "serde", "winnow 1.0.1", @@ -860,9 +860,9 @@ dependencies = [ [[package]] name = "alloy-sol-types" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "384cf252de0db2dec52821eac037a7f57e2aa33fe5b900ce6fe39973402341f1" +checksum = "d96e74d6213180f78dbdccddce8af02a639c160c94b0a543fa35c77c58b8a7fc" dependencies = [ "alloy-json-abi", "alloy-primitives", @@ -901,7 +901,7 @@ checksum = "0e79a2c1793afc61eed9ca0963da370a988b27d2bbfa67c78013e262d47424c4" dependencies = [ "alloy-json-rpc", "alloy-transport", - "itertools 0.13.0", + "itertools 0.14.0", "reqwest", "serde_json", "tower", @@ -2400,7 +2400,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -2906,7 +2906,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 1.0.109", + "syn 2.0.118", ] [[package]] @@ -3360,7 +3360,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -5120,7 +5120,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -6954,6 +6954,16 @@ dependencies = [ "quote", ] +[[package]] +name = "proc-macro-error-attr3" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "82366fd7d8b7a440d66d13418820c69df9b3908bcb1a0476d7f5ce5d12f5a04d" +dependencies = [ + "proc-macro2", + "quote", +] + [[package]] name = "proc-macro-error2" version = "2.0.1" @@ -6963,6 +6973,17 @@ dependencies = [ "proc-macro-error-attr2", "proc-macro2", "quote", +] + +[[package]] +name = "proc-macro-error3" +version = "3.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b511283ea8a74b4b39447b128c5d00f03a356b7424554b13e298a5550100d9ac" +dependencies = [ + "proc-macro-error-attr3", + "proc-macro2", + "quote", "syn 2.0.118", ] @@ -7067,7 +7088,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.10.5", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.118", @@ -11208,7 +11229,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11267,7 +11288,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 0.26.11", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11288,7 +11309,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -12054,9 +12075,9 @@ dependencies = [ [[package]] name = "syn-solidity" -version = "1.6.0" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec005042c7d952febc1a3ef5b0f6674e9054aa836877a31c90b20e25b3d31744" +checksum = "083be3061e64d362cbe6ef12cfe1307ba3884326d8856448fe8a120fa2c44ebf" dependencies = [ "paste", "proc-macro2", @@ -12158,7 +12179,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -13437,7 +13458,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 20951a3d1ec..b2c9f8f6384 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -433,9 +433,9 @@ revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default revm-inspectors = "0.41.0" # eth -alloy-dyn-abi = "1.6.0" -alloy-primitives = { version = "1.6.0", default-features = false, features = ["map-foldhash"] } -alloy-sol-types = { version = "1.6.0", default-features = false } +alloy-dyn-abi = "1.6.1" +alloy-primitives = { version = "1.6.1", default-features = false, features = ["map-foldhash"] } +alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } From 3ce0b4ecb545b4a1a40d685b519da1100b845e86 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 16 Jul 2026 18:09:58 +0200 Subject: [PATCH 010/148] fix(trie): group proof targets by exact subtrie prefix (#26357) --- crates/trie/trie/src/proof_v2/mod.rs | 191 +++++++++++++++++------- crates/trie/trie/src/proof_v2/target.rs | 110 ++++++-------- 2 files changed, 185 insertions(+), 116 deletions(-) diff --git a/crates/trie/trie/src/proof_v2/mod.rs b/crates/trie/trie/src/proof_v2/mod.rs index 7a8cd904a89..5068d805fc6 100644 --- a/crates/trie/trie/src/proof_v2/mod.rs +++ b/crates/trie/trie/src/proof_v2/mod.rs @@ -205,15 +205,6 @@ where let (mut lower, mut upper) = targets.current(); - debug_assert!(self.retained_proofs.last().is_none_or( - |ProofTrieNodeV2 { path: last_retained_path, .. }| { - depth_first::cmp(path, last_retained_path) == Ordering::Greater - } - ), - "should_retain called with path {path:?} which is not after previously retained node {:?} in depth-first order", - self.retained_proofs.last().map(|n| n.path), - ); - loop { // If the node in question is a prefix of the target then we do not iterate targets // further. @@ -687,7 +678,7 @@ where &mut self, value_encoder: &mut VE, targets: &mut Option>, - hashed_cursor_current: &mut Option<(Nibbles, VE::DeferredEncoder)>, + hashed_cursor_state: &mut HashedCursorState, lower_bound: Nibbles, upper_bound: Option, ) -> Result<(), StateProofError> { @@ -701,28 +692,33 @@ where (key, val) }; - // If the cursor hasn't been used, or the last iterated key is prior to this range's - // key range, then seek forward to at least the first key. - if hashed_cursor_current.as_ref().is_none_or(|(key, _)| key < &lower_bound) { + // If the cursor hasn't been used, or the last iterated key is prior to this range's key + // range, then seek forward to at least the first key. + if hashed_cursor_state.needs_seek_to(&lower_bound) { trace!( target: TRACE_TARGET, - current=?hashed_cursor_current.as_ref().map(|(k, _)| k), + current=?hashed_cursor_state.path(), "Seeking hashed cursor to meet lower bound", ); let lower_key = B256::right_padding_from(&lower_bound.pack()); - *hashed_cursor_current = - self.hashed_cursor.seek(lower_key)?.map(&mut map_hashed_cursor_entry); + *hashed_cursor_state = HashedCursorState::seeked( + lower_bound, + self.hashed_cursor.seek(lower_key)?.map(&mut map_hashed_cursor_entry), + ); } // Loop over all keys in the range, calling `push_leaf` on each. - while let Some((key, _)) = hashed_cursor_current.as_ref() && - upper_bound.is_none_or(|upper_bound| key < &upper_bound) + while hashed_cursor_state + .path() + .is_some_and(|key| upper_bound.is_none_or(|upper_bound| key < &upper_bound)) { - let (key, val) = - core::mem::take(hashed_cursor_current).expect("while-let checks for Some"); + let (key, val) = hashed_cursor_state.take(); self.push_leaf(targets, key, val)?; - *hashed_cursor_current = self.hashed_cursor.next()?.map(&mut map_hashed_cursor_entry); + *hashed_cursor_state = HashedCursorState::seeked( + key, + self.hashed_cursor.next()?.map(&mut map_hashed_cursor_entry), + ); } trace!(target: TRACE_TARGET, "No further keys within range"); @@ -927,8 +923,10 @@ where // If the trie cursor is seeked to a branch whose leaves have already been processed // then we can't use it, instead we seek forward and try again. if trie_cursor_path < uncalculated_lower_bound { - *trie_cursor_state = - TrieCursorState::seeked(self.trie_cursor_seek(*uncalculated_lower_bound)?); + *trie_cursor_state = TrieCursorState::seeked( + *uncalculated_lower_bound, + self.trie_cursor_seek(*uncalculated_lower_bound)?, + ); // Having just seeked forward we need to check if the cursor is now exhausted, // extracting the new path at the same time. @@ -1244,7 +1242,8 @@ where // trie cursor to the next cached node at-or-after `child_path`. if trie_cursor_state.path().is_some_and(|path| path < &child_path) { trace!(target: TRACE_TARGET, ?child_path, "Seeking trie cursor to child path"); - *trie_cursor_state = TrieCursorState::seeked(self.trie_cursor_seek(child_path)?); + *trie_cursor_state = + TrieCursorState::seeked(child_path, self.trie_cursor_seek(child_path)?); } // If the next cached branch node is a child of `child_path` then we can assume it is @@ -1309,7 +1308,7 @@ where &mut self, value_encoder: &mut VE, trie_cursor_state: &mut TrieCursorState, - hashed_cursor_current: &mut Option<(Nibbles, VE::DeferredEncoder)>, + hashed_cursor_state: &mut HashedCursorState, sub_trie_targets: SubTrieTargets<'a>, ) -> Result<(), StateProofError> { let sub_trie_upper_bound = sub_trie_targets.upper_bound(); @@ -1330,12 +1329,24 @@ where debug_assert!(self.child_stack.is_empty()); // `next_uncached_key_range`, which will be called in the loop below, expects the trie - // cursor to have already been seeked. If it's not yet seeked, or seeked to a prior node, - // then we seek it to the prefix (the first possible node) to initialize it. - if trie_cursor_state.before(&sub_trie_targets.prefix) { - trace!(target: TRACE_TARGET, "Doing initial seek of trie cursor"); - *trie_cursor_state = - TrieCursorState::seeked(self.trie_cursor_seek(sub_trie_targets.prefix)?); + // cursor to have already been seeked. The trie cursor is forward-only, but exact sub-trie + // chunks can overlap previous chunks, so reset it if this seek needs to move backwards. + if trie_cursor_state.needs_reset_before_seek(&sub_trie_targets.prefix) { + trace!(target: TRACE_TARGET, "Resetting trie cursor before sub-trie"); + self.trie_cursor.reset(); + *trie_cursor_state = TrieCursorState::unseeked(); + } + + trace!(target: TRACE_TARGET, "Doing initial seek of trie cursor"); + *trie_cursor_state = TrieCursorState::seeked( + sub_trie_targets.prefix, + self.trie_cursor_seek(sub_trie_targets.prefix)?, + ); + + if hashed_cursor_state.needs_reset_before_seek(&sub_trie_targets.prefix) { + trace!(target: TRACE_TARGET, "Resetting hashed cursor before sub-trie"); + self.hashed_cursor.reset(); + *hashed_cursor_state = HashedCursorState::unseeked(); } // `uncalculated_lower_bound` tracks the lower bound of node paths which have yet to be @@ -1385,20 +1396,20 @@ where self.calculate_key_range( value_encoder, &mut targets, - hashed_cursor_current, + hashed_cursor_state, calc_lower_bound, calc_upper_bound, )?; - // Once outside `calculate_key_range`, `hashed_cursor_current` will be at the first key - // after the range. + // Once outside `calculate_key_range`, `hashed_cursor_state` will be at the first key + // after the range, or exhausted. // - // If the `hashed_cursor_current` is None (exhausted), or not within the range of the + // If the hashed cursor is exhausted, or not within the range of the // sub-trie, then there are no more keys at all, meaning the trie couldn't possibly have // more data and we should complete computation. - if hashed_cursor_current - .as_ref() - .is_none_or(|(key, _)| !key.starts_with(&sub_trie_targets.prefix)) + if hashed_cursor_state + .path() + .is_none_or(|key| !key.starts_with(&sub_trie_targets.prefix)) { break; } @@ -1484,7 +1495,7 @@ where // Initialize the variables which track the state of the two cursors. Both indicate the // cursors are unseeked. let mut trie_cursor_state = TrieCursorState::unseeked(); - let mut hashed_cursor_current: Option<(Nibbles, VE::DeferredEncoder)> = None; + let mut hashed_cursor_state = HashedCursorState::unseeked(); // Divide targets into chunks, each chunk corresponding to a different sub-trie within the // overall trie, and handle all proofs within that sub-trie. @@ -1492,7 +1503,7 @@ where if let Err(err) = self.proof_subtrie( value_encoder, &mut trie_cursor_state, - &mut hashed_cursor_current, + &mut hashed_cursor_state, sub_trie_targets, ) { self.clear_computation_state(); @@ -1505,6 +1516,8 @@ where retained_proofs_len = ?self.retained_proofs.len(), "proof_inner: returning", ); + self.retained_proofs.sort_unstable_by(|a, b| depth_first::cmp(&a.path, &b.path)); + self.retained_proofs.dedup_by(|a, b| a.path == b.path); Ok(core::mem::take(&mut self.retained_proofs)) } @@ -1564,7 +1577,7 @@ where // Initialize the variables which track the state of the two cursors. Both indicate the // cursors are unseeked. let mut trie_cursor_state = TrieCursorState::unseeked(); - let mut hashed_cursor_current: Option<(Nibbles, VE::DeferredEncoder)> = None; + let mut hashed_cursor_state = HashedCursorState::unseeked(); static EMPTY_TARGETS: [ProofV2Target; 0] = []; let sub_trie_targets = @@ -1573,7 +1586,7 @@ where if let Err(err) = self.proof_subtrie( value_encoder, &mut trie_cursor_state, - &mut hashed_cursor_current, + &mut hashed_cursor_state, sub_trie_targets, ) { self.clear_computation_state(); @@ -1739,8 +1752,8 @@ enum TrieCursorState { Available(Nibbles, BranchNodeCompact), /// Cursor is seeked to this path, but the node has been used. Taken(Nibbles), - /// Cursor has been exhausted. - Exhausted, + /// Cursor has been exhausted after seeking from the given lower bound. + Exhausted(Nibbles), } impl TrieCursorState { @@ -1750,8 +1763,8 @@ impl TrieCursorState { } /// Creates a [`Self`] based on an entry returned from the cursor itself. - fn seeked(entry: Option<(Nibbles, BranchNodeCompact)>) -> Self { - entry.map_or(Self::Exhausted, |(path, node)| Self::Available(path, node)) + fn seeked(key: Nibbles, entry: Option<(Nibbles, BranchNodeCompact)>) -> Self { + entry.map_or(Self::Exhausted(key), |(path, node)| Self::Available(path, node)) } /// Returns the path the cursor is seeked to, or None if it's exhausted. @@ -1763,16 +1776,16 @@ impl TrieCursorState { match self { Self::Unseeked => panic!("cursor is unseeked"), Self::Available(path, _) | Self::Taken(path) => Some(path), - Self::Exhausted => None, + Self::Exhausted(_) => None, } } - /// Returns true if the cursor is unseeked, or is seeked to a node prior to the given one. - fn before(&self, path: &Nibbles) -> bool { + /// Returns true if seeking to `key` requires resetting the forward-only cursor. + fn needs_reset_before_seek(&self, key: &Nibbles) -> bool { match self { - Self::Unseeked => true, - Self::Available(seeked_to, _) | Self::Taken(seeked_to) => path < seeked_to, - Self::Exhausted => false, + Self::Unseeked => false, + Self::Available(path, _) | Self::Taken(path) => path > key, + Self::Exhausted(exhausted_at) => exhausted_at > key, } } @@ -1791,6 +1804,62 @@ impl TrieCursorState { } } +/// Used to track the state of the hashed cursor, including the path that established exhaustion. +enum HashedCursorState { + /// The initial state of the cursor, indicating it's never been seeked. + Unseeked, + /// Cursor is seeked to this path and the value has not been used yet. + Available(Nibbles, V), + /// Cursor has been exhausted at or after the given path. + Exhausted(Nibbles), +} + +impl HashedCursorState { + /// Creates a [`Self::Unseeked`] state. + const fn unseeked() -> Self { + Self::Unseeked + } + + /// Creates a [`Self`] based on an entry returned from the cursor itself. + fn seeked(key: Nibbles, entry: Option<(Nibbles, V)>) -> Self { + entry.map_or(Self::Exhausted(key), |(path, value)| Self::Available(path, value)) + } + + /// Returns the path the cursor is seeked to, or None if it's unseeked or exhausted. + const fn path(&self) -> Option<&Nibbles> { + match self { + Self::Available(path, _) => Some(path), + Self::Unseeked | Self::Exhausted(_) => None, + } + } + + /// Returns true if the cursor must seek to be usable for a range starting at `key`. + fn needs_seek_to(&self, key: &Nibbles) -> bool { + match self { + Self::Unseeked => true, + Self::Available(path, _) => path < key, + Self::Exhausted(exhausted_at) => exhausted_at > key, + } + } + + /// Returns true if seeking to `key` requires resetting the forward-only cursor. + fn needs_reset_before_seek(&self, key: &Nibbles) -> bool { + match self { + Self::Unseeked => false, + Self::Available(path, _) => path > key, + Self::Exhausted(exhausted_at) => exhausted_at > key, + } + } + + /// Takes the path and value from a [`Self::Available`]. Panics if not [`Self::Available`]. + fn take(&mut self) -> (Nibbles, V) { + match core::mem::replace(self, Self::Unseeked) { + Self::Available(path, value) => (path, value), + _ => panic!("take called on non-Available hashed cursor state"), + } + } +} + /// Describes the state of the currently cached branch node (if any). enum PopCachedBranchOutcome { /// Cached branch has been popped from the `cached_branch_stack` and is ready to be used. @@ -2069,6 +2138,24 @@ mod tests { } } + #[test] + fn test_exact_subtrie_targets_with_root_target() { + reth_tracing::init_test_tracing(); + + let slot_80 = B256::right_padding_from(&[0x80]); + let slot_82 = B256::right_padding_from(&[0x82]); + let slot_f0 = B256::right_padding_from(&[0xf0]); + let storage = BTreeMap::from([ + (slot_80, U256::from(1)), + (slot_82, U256::from(2)), + (slot_f0, U256::from(3)), + ]); + let targets = [ProofV2Target::new(B256::ZERO), ProofV2Target::new(slot_80).with_min_len(2)]; + + let harness = ProofTestHarness::new(storage); + harness.assert_proof(targets).expect("Proof generation failed"); + } + #[test] fn test_big_trie() { use rand::prelude::*; diff --git a/crates/trie/trie/src/proof_v2/target.rs b/crates/trie/trie/src/proof_v2/target.rs index 14be86c7ac9..2077545a2a4 100644 --- a/crates/trie/trie/src/proof_v2/target.rs +++ b/crates/trie/trie/src/proof_v2/target.rs @@ -59,51 +59,23 @@ impl<'a> SubTrieTargets<'a> { pub(crate) fn iter_sub_trie_targets( targets: &mut [ProofV2Target], ) -> impl Iterator> { - // First sort by the sub-trie prefix of each target, falling back to the `min_len` in cases - // where the sub-trie prefixes are equal (to differentiate targets which match the root node and - // those which don't). + // Sort globally by sub-trie prefix, then by target key. This makes equal-prefix targets + // contiguous and already ordered for the `ProofCalculator`. targets.sort_unstable_by(|a, b| { - sub_trie_prefix(a).cmp(&sub_trie_prefix(b)).then_with(|| a.min_len.cmp(&b.min_len)) + sub_trie_prefix(a).cmp(&sub_trie_prefix(b)).then_with(|| a.key_nibbles.cmp(&b.key_nibbles)) }); - // We now chunk targets, such that each chunk contains all targets belonging to the same - // sub-trie. We are taking advantage of the following properties: - // - // - The first target in the chunk has the shortest sub-trie prefix (see previous sorting step). - // - // - The upper bound of the first target in the chunk's sub-trie will therefore be the upper - // bound of the whole chunk. - // - For example, given a chunk with sub-trie prefixes [0x2, 0x2f, 0x2fa], the upper bounds - // will be [0x3, 0x3, 0x2fb]. Note that no target could match a trie node with path equal - // to or greater than 0x3. - // - // - If a target's sub-trie's prefix does not lie within the bounds of the current chunk, then - // that target must be the first target of the next chunk, lying in a separate sub-trie. - // - Example: given sub-trie prefixes of [0x2, 0x2fa, 0x4c, 0x4ce, 0x4e], we would end up - // with the following chunks: - // - [0x2, 0x2fa] w/ upper bound 0x3 - // - [0x4c 0x4ce] w/ upper bound 0x4d - // - [0x4e] w/ upper bound 0x4f - let mut upper_bound = targets.first().and_then(|t| sub_trie_upper_bound(&sub_trie_prefix(t))); - let target_chunks = targets.chunk_by_mut(move |_, next| { - if let Some(some_upper_bound) = upper_bound { - let prefix = sub_trie_prefix(next); - let same_chunk = prefix < some_upper_bound; - if !same_chunk { - upper_bound = sub_trie_upper_bound(&prefix); - } - same_chunk - } else { - true - } - }); + // Chunk targets by exact sub-trie prefix. Prefixes nested below a previous prefix are still + // processed as their own sub-trie. + let target_chunks = + targets.chunk_by_mut(|current, next| sub_trie_prefix(current) == sub_trie_prefix(next)); - // Map the chunks to the return type. Within each chunk we want targets to be sorted by their - // key, as that will be the order they are checked by the `ProofCalculator`. + // Map the chunks to the return type. target_chunks.map(move |targets| { let prefix = sub_trie_prefix(&targets[0]); - let retain_root = targets[0].min_len == 0; - targets.sort_unstable_by_key(|target| target.key_nibbles); + // Targets with `min_len` 0 and 1 share the empty prefix, so key ordering cannot indicate + // whether the root should be retained. + let retain_root = targets.iter().any(|target| target.min_len == 0); SubTrieTargets { prefix, targets, retain_root } }) } @@ -179,21 +151,21 @@ mod tests { ("4", vec!["4040404040404040404040404040404040404040404040404040404040404040"]), ], ), - // Case 6: Targets with different min_len values in same sub-trie + // Case 6: Targets with different min_len values in different sub-tries ( vec![ ProofV2Target::new(B256::repeat_byte(0x20)).with_min_len(2), ProofV2Target::new(B256::repeat_byte(0x2f)).with_min_len(3), ], - vec![( - "2", - vec![ - "2020202020202020202020202020202020202020202020202020202020202020", - "2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f", - ], - )], + vec![ + ("2", vec!["2020202020202020202020202020202020202020202020202020202020202020"]), + ( + "2f", + vec!["2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f"], + ), + ], ), - // Case 7: More complex chunking with multiple sub-tries + // Case 7: More complex chunking with nested sub-trie prefixes ( vec![ ProofV2Target::new(B256::repeat_byte(0x20)).with_min_len(2), @@ -203,19 +175,18 @@ mod tests { ProofV2Target::new(B256::repeat_byte(0x4e)).with_min_len(3), ], vec![ + ("2", vec!["2020202020202020202020202020202020202020202020202020202020202020"]), ( - "2", - vec![ - "2020202020202020202020202020202020202020202020202020202020202020", - "2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f", - ], + "2f2", + vec!["2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f2f"], ), ( "4c", - vec![ - "4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c", - "4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c", - ], + vec!["4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c"], + ), + ( + "4c4", + vec!["4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c4c"], ), ( "4e", @@ -243,13 +214,10 @@ mod tests { ProofV2Target::new(B256::repeat_byte(0x20)).with_min_len(2), ProofV2Target::new(B256::repeat_byte(0x40)).with_min_len(1), ], - vec![( - "", - vec![ - "2020202020202020202020202020202020202020202020202020202020202020", - "4040404040404040404040404040404040404040404040404040404040404040", - ], - )], + vec![ + ("", vec!["4040404040404040404040404040404040404040404040404040404040404040"]), + ("2", vec!["2020202020202020202020202020202020202020202020202020202020202020"]), + ], ), ]; @@ -299,4 +267,18 @@ mod tests { } } } + + #[test] + fn test_iter_sub_trie_targets_retain_root_after_key_sort() { + let mut targets = [ + ProofV2Target::new(B256::repeat_byte(0x40)), + ProofV2Target::new(B256::repeat_byte(0x20)).with_min_len(1), + ]; + + let sub_tries = iter_sub_trie_targets(&mut targets).collect::>(); + + assert_eq!(sub_tries.len(), 1); + assert_eq!(sub_tries[0].targets[0].key(), B256::repeat_byte(0x20)); + assert!(sub_tries[0].retain_root); + } } From 091caca633eefcfe86300854675106bd8f39e33f Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 17 Jul 2026 12:50:47 +0200 Subject: [PATCH 011/148] fix(engine): avoid rewinding preserved sparse trie anchors (#26422) --- .../tree/src/tree/state_root_strategy/mod.rs | 48 +++++++++++++++---- 1 file changed, 39 insertions(+), 9 deletions(-) diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 7db42af1e35..d778f23de10 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -775,9 +775,21 @@ fn published_sparse_trie_anchor_hash( return sparse_trie_anchor_hash } - pending_sparse_trie_prune_blocks - .and_then(|blocks| blocks.last().map(|block| block.recovered_block().parent_hash())) - .unwrap_or(sparse_trie_anchor_hash) + let Some(prune_blocks) = pending_sparse_trie_prune_blocks else { + return sparse_trie_anchor_hash + }; + let Some(oldest_prune_block) = prune_blocks.last() else { return sparse_trie_anchor_hash }; + + // Prune blocks contain the complete in-memory parent chain from newest to oldest, with the + // oldest block's parent being the persisted tip. A fresh trie can be anchored to an in-memory + // block ahead of that tip. If that anchor is still in the prune range, publishing the + // persisted tip as the new anchor would expand the trie's claimed coverage backwards even + // though pruning cannot reveal those paths. + if prune_blocks.iter().any(|block| block.recovered_block().hash() == sparse_trie_anchor_hash) { + return sparse_trie_anchor_hash + } + + oldest_prune_block.recovered_block().parent_hash() } impl StateRootStrategy for DefaultStateRootStrategy @@ -1358,19 +1370,37 @@ mod tests { } #[test] - fn published_sparse_trie_anchor_uses_prune_anchor_for_reused_trie() { - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect(); - let reused_anchor_hash = B256::with_last_byte(0xaa); + fn published_sparse_trie_anchor_advances_to_prune_anchor() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..5).collect(); + let reused_anchor_hash = blocks[0].recovered_block().hash(); + let expected_prune_anchor = blocks[1].recovered_block().hash(); + let prune_blocks: Vec<_> = blocks.into_iter().skip(2).rev().collect(); assert_eq!( - published_sparse_trie_anchor_hash(reused_anchor_hash, true, Some(&blocks)), - blocks.last().unwrap().recovered_block().parent_hash() + published_sparse_trie_anchor_hash(reused_anchor_hash, true, Some(&prune_blocks)), + expected_prune_anchor + ); + } + + #[test] + fn published_sparse_trie_anchor_does_not_move_backwards_when_anchor_is_in_prune_range() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..5).collect(); + let reused_anchor_hash = blocks[2].recovered_block().hash(); + let mut prune_blocks = blocks; + prune_blocks.reverse(); + let prune_anchor = prune_blocks.last().unwrap().recovered_block().parent_hash(); + + assert_ne!(reused_anchor_hash, prune_anchor); + assert_eq!( + published_sparse_trie_anchor_hash(reused_anchor_hash, true, Some(&prune_blocks)), + reused_anchor_hash ); } #[test] fn published_sparse_trie_anchor_keeps_parent_for_fresh_trie() { - let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect(); + let mut blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect(); + blocks.reverse(); let parent_hash = B256::with_last_byte(0xaa); assert_eq!( From 2382a2b05a0f15ab5469ea1db152c944f3736f6c Mon Sep 17 00:00:00 2001 From: Francis Li Date: Fri, 17 Jul 2026 04:15:29 -0700 Subject: [PATCH 012/148] feat(engine): pass parent header to post-execution hook (#26420) --- crates/engine/primitives/src/lib.rs | 5 ++++- crates/engine/tree/src/tree/payload_validator.rs | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/crates/engine/primitives/src/lib.rs b/crates/engine/primitives/src/lib.rs index 55196ada18e..7855b803871 100644 --- a/crates/engine/primitives/src/lib.rs +++ b/crates/engine/primitives/src/lib.rs @@ -16,7 +16,7 @@ use reth_payload_primitives::{ EngineApiMessageVersion, EngineObjectValidationError, InvalidPayloadAttributesError, NewPayloadError, PayloadAttributes, PayloadOrAttributes, PayloadTypes, }; -use reth_primitives_traits::{Block, RecoveredBlock, SealedBlock}; +use reth_primitives_traits::{Block, RecoveredBlock, SealedBlock, SealedHeader}; use reth_storage_api::{errors::ProviderResult, StateProviderBox}; use reth_trie_common::HashedPostState; use serde::{de::DeserializeOwned, Serialize}; @@ -217,6 +217,8 @@ pub trait PayloadValidator: Send + Sync + Unpin + 'static { /// `state_updates` lazily yields the block's hashed post-state; call it only if the /// implementation needs the executed state changes (the L1 default does not). /// + /// `parent_header` is the parent header the engine resolved for the block. + /// /// `parent_state` lazily builds the overlay-aware provider for the block's parent that the /// engine used for execution — resolving even a not-yet-canonical in-memory parent. It is only /// built if the implementation needs it (the L1 default does not). @@ -224,6 +226,7 @@ pub trait PayloadValidator: Send + Sync + Unpin + 'static { &self, _state_updates: impl FnOnce() -> &'a HashedPostState, _block: &RecoveredBlock, + _parent_header: &SealedHeader<::Header>, _parent_state: impl FnOnce() -> ProviderResult, ) -> Result<(), InsertBlockErrorKind> where diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index f8efd2cc093..c2c3db449a6 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -788,6 +788,7 @@ where self.validator.validate_block_post_execution_with_hashed_state( || hashed_state.get(), &block, + &parent_block, || provider_builder.build(), ) }); @@ -823,6 +824,7 @@ where self.validator.validate_block_post_execution_with_hashed_state( || hashed_state.get(), &block, + &parent_block, || provider_builder.build(), ) }); From 234b353e581e7460a11abd0f0805cd7fc05e4bf7 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Fri, 17 Jul 2026 17:04:47 +0300 Subject: [PATCH 013/148] chore: bump version to 2.4.1 (#26426) --- Cargo.lock | 226 +++++++++++++++++++-------------------- Cargo.toml | 2 +- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 115 insertions(+), 115 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 106b84b57b1..098f32710b9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3202,7 +3202,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.4.0" +version = "2.4.1" dependencies = [ "clap", "ef-tests", @@ -3210,7 +3210,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3663,7 +3663,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "eyre", @@ -3796,7 +3796,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "clap", @@ -7568,7 +7568,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7618,7 +7618,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7644,7 +7644,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7691,7 +7691,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7724,7 +7724,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7744,7 +7744,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-genesis", "clap", @@ -7756,7 +7756,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7846,7 +7846,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.4.0" +version = "2.4.1" dependencies = [ "reth-tasks", "tokio", @@ -7855,7 +7855,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -7909,7 +7909,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "eyre", @@ -7927,7 +7927,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -7940,7 +7940,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7954,7 +7954,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7978,7 +7978,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8014,7 +8014,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8042,7 +8042,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8073,7 +8073,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8089,7 +8089,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8115,7 +8115,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8140,7 +8140,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8168,7 +8168,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8206,7 +8206,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8262,7 +8262,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.4.0" +version = "2.4.1" dependencies = [ "aes", "alloy-primitives", @@ -8288,7 +8288,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8310,7 +8310,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8336,7 +8336,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8405,7 +8405,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8434,7 +8434,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8459,7 +8459,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "bytes", @@ -8477,7 +8477,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8505,7 +8505,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.4.0" +version = "2.4.1" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8515,7 +8515,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8553,7 +8553,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8580,7 +8580,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8620,7 +8620,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.4.0" +version = "2.4.1" dependencies = [ "clap", "eyre", @@ -8643,7 +8643,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8659,7 +8659,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8675,7 +8675,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.7", @@ -8688,7 +8688,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8717,7 +8717,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8738,7 +8738,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "rayon", @@ -8748,7 +8748,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8773,7 +8773,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8798,7 +8798,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "fixed-cache", @@ -8816,7 +8816,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-evm", "alloy-primitives", @@ -8828,7 +8828,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8849,7 +8849,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8894,7 +8894,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "eyre", @@ -8925,7 +8925,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8942,7 +8942,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.4.0" +version = "2.4.1" dependencies = [ "serde", "serde_json", @@ -8951,7 +8951,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8982,7 +8982,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.4.0" +version = "2.4.1" dependencies = [ "bytes", "futures", @@ -9004,7 +9004,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.4.0" +version = "2.4.1" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -9021,7 +9021,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.4.0" +version = "2.4.1" dependencies = [ "bindgen", "cc", @@ -9029,7 +9029,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.4.0" +version = "2.4.1" dependencies = [ "futures", "metrics", @@ -9041,7 +9041,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "ipnet", @@ -9049,7 +9049,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.4.0" +version = "2.4.1" dependencies = [ "futures-util", "if-addrs", @@ -9063,7 +9063,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9126,7 +9126,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9150,7 +9150,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9174,7 +9174,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9191,7 +9191,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9204,7 +9204,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.4.0" +version = "2.4.1" dependencies = [ "anyhow", "bincode", @@ -9222,7 +9222,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9245,7 +9245,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9316,7 +9316,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9372,7 +9372,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9441,7 +9441,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9464,7 +9464,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9487,7 +9487,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.4.0" +version = "2.4.1" dependencies = [ "bytes", "eyre", @@ -9516,7 +9516,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9527,7 +9527,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9550,7 +9550,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.4.0" +version = "2.4.1" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9561,7 +9561,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9585,7 +9585,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9594,7 +9594,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9636,7 +9636,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9689,7 +9689,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9720,7 +9720,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "arbitrary", @@ -9740,7 +9740,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9756,7 +9756,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -9836,7 +9836,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-genesis", @@ -9866,7 +9866,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-network", @@ -9924,7 +9924,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-evm", @@ -9944,7 +9944,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -9964,7 +9964,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10000,7 +10000,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10046,7 +10046,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10098,7 +10098,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-rpc-types-engine", "http", @@ -10115,7 +10115,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10145,7 +10145,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10207,7 +10207,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10241,7 +10241,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "arbitrary", @@ -10257,7 +10257,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "assert_matches", @@ -10280,7 +10280,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "clap", @@ -10298,7 +10298,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10325,7 +10325,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10341,7 +10341,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10370,7 +10370,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.4.0" +version = "2.4.1" dependencies = [ "crossbeam-utils", "dashmap", @@ -10390,7 +10390,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10406,7 +10406,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.4.0" +version = "2.4.1" dependencies = [ "tokio", "tokio-stream", @@ -10415,7 +10415,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.4.0" +version = "2.4.1" dependencies = [ "clap", "eyre", @@ -10434,7 +10434,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.4.0" +version = "2.4.1" dependencies = [ "base64 0.22.1", "clap", @@ -10452,7 +10452,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10503,7 +10503,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10534,7 +10534,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10566,7 +10566,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10596,7 +10596,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10624,7 +10624,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.4.0" +version = "2.4.1" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index b2c9f8f6384..6f108fe3cdc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.4.0" +version = "2.4.1" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index 151debb8756..bb8a8fba54b 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.4.0', + text: 'v2.4.1', items: [ { text: 'Releases', From 83d54cca6c379a1b7fe7bab3cab4e223d34f3413 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Fri, 17 Jul 2026 16:45:21 +0200 Subject: [PATCH 014/148] refactor(builder): lease resources (#26427) --- crates/engine/tree/src/tree/mod.rs | 11 +- .../engine/tree/src/tree/payload_validator.rs | 126 ++++++------ crates/engine/tree/src/tree/tests.rs | 2 +- crates/payload/basic/src/lib.rs | 13 +- crates/payload/builder/src/lib.rs | 4 +- crates/payload/builder/src/service.rs | 183 +++++++++++++++--- 6 files changed, 238 insertions(+), 101 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index d78cf0838a5..6664f49c55d 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -3305,13 +3305,7 @@ where // payloadAttributes is not null and the forkchoice state has been updated successfully. // The build process is specified in the Payload building section. - let cache = if self.config.share_execution_cache_with_payload_builder() { - self.payload_validator.cache_for(state.head_block_hash) - } else { - None - }; - - let state_root_handle = self.payload_validator.payload_state_root_handle_for( + let resources = self.payload_validator.payload_builder_resources( state.head_block_hash, head, attributes.timestamp(), @@ -3323,8 +3317,7 @@ where let pending_payload_id = self.payload_builder.send_new_payload(BuildNewPayload { parent_hash: state.head_block_hash, attributes, - cache, - state_root_handle, + resources, }); // Client software MUST respond to this method call in the following way: diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index c2c3db449a6..4f8a59cd3ba 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -136,7 +136,8 @@ use reth_evm::{ block::BlockExecutor, execute::ExecutableTxFor, ConfigureEvm, EvmEnvFor, ExecutionCtxFor, OnStateHook, SpecFor, }; -use reth_execution_cache::{CacheFillMode, CacheStats, SavedCache}; +use reth_execution_cache::{CacheFillMode, CacheStats}; +use reth_payload_builder::PayloadBuilderResources; use reth_payload_primitives::{ BuiltPayload, BuiltPayloadExecutedBlock, InvalidPayloadAttributesError, NewPayloadError, PayloadTypes, @@ -1431,6 +1432,57 @@ where .with_state_trie_overlay_manager(state.tree_state.state_trie_overlays.clone()) } + /// Prepares the optional payload-builder state-root handle through the installed + /// [`StateRootStrategy`]. + fn payload_state_root_handle_for( + &self, + parent_hash: B256, + parent_header: &N::BlockHeader, + timestamp: u64, + state: &mut EngineApiTreeState, + ) -> Option { + let provider_builder = match self.state_provider_builder(parent_hash, state) { + Ok(Some(provider_builder)) => provider_builder, + Ok(None) => return None, + Err(err) => { + warn!( + target: "engine::tree::payload_validator", + %err, + %parent_hash, + "failed to prepare payload-builder state-root provider" + ); + return None + } + }; + let overlay_factory = OverlayStateProviderFactory::new( + self.provider.clone(), + Self::overlay_builder_for_parent(parent_hash, state, self.changeset_cache.clone()), + ); + + match self.state_root_strategy.prepare_payload_builder(PayloadStateRootJobContext::new( + &self.runtime, + &self.state_trie_overlays, + parent_hash, + parent_header, + timestamp, + state, + provider_builder, + overlay_factory, + &self.config, + )) { + Ok(handle) => handle, + Err(err) => { + warn!( + target: "engine::tree::payload_validator", + %err, + %parent_hash, + "failed to prepare payload-builder state-root job" + ); + None + } + } + } + /// Spawns a background task to compute and sort trie data for the executed block. /// /// This function creates a [`LazyTrieData`] handle and spawns a blocking task that: @@ -1689,24 +1741,16 @@ pub trait EngineValidator< block: BuiltPayloadExecutedBlock, ) -> ProviderResult>; - /// Returns [`SavedCache`] for the given block hash. - fn cache_for(&self, _block_hash: B256) -> Option; - - /// Prepares the optional payload-builder state-root handle through the installed - /// [`StateRootStrategy`]. + /// Prepares the resources loaned to a payload builder job. /// - /// Returns `None` when the strategy declines, in which case the payload builder computes - /// the state root itself. - /// - /// `timestamp` is the timestamp of the payload being built, taken from the payload - /// attributes. - fn payload_state_root_handle_for( + /// `timestamp` is taken from the payload attributes. + fn payload_builder_resources( &self, parent_hash: B256, parent_header: &N::BlockHeader, timestamp: u64, state: &mut EngineApiTreeState, - ) -> Option; + ) -> PayloadBuilderResources; } impl EngineValidator for BasicEngineValidator @@ -1786,57 +1830,21 @@ where )) } - fn cache_for(&self, block_hash: B256) -> Option { - Some(self.payload_processor.cache_for(block_hash)) - } - - fn payload_state_root_handle_for( + fn payload_builder_resources( &self, parent_hash: B256, parent_header: &N::BlockHeader, timestamp: u64, state: &mut EngineApiTreeState, - ) -> Option { - let provider_builder = match self.state_provider_builder(parent_hash, state) { - Ok(Some(provider_builder)) => provider_builder, - Ok(None) => return None, - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - %parent_hash, - "failed to prepare payload-builder state-root provider" - ); - return None - } - }; - let overlay_factory = OverlayStateProviderFactory::new( - self.provider.clone(), - Self::overlay_builder_for_parent(parent_hash, state, self.changeset_cache.clone()), - ); - - match self.state_root_strategy.prepare_payload_builder(PayloadStateRootJobContext::new( - &self.runtime, - &self.state_trie_overlays, - parent_hash, - parent_header, - timestamp, - state, - provider_builder, - overlay_factory, - &self.config, - )) { - Ok(handle) => handle, - Err(err) => { - warn!( - target: "engine::tree::payload_validator", - %err, - %parent_hash, - "failed to prepare payload-builder state-root job" - ); - None - } - } + ) -> PayloadBuilderResources { + let execution_cache = self + .config + .share_execution_cache_with_payload_builder() + .then(|| self.payload_processor.cache_for(parent_hash)); + let state_root_handle = + self.payload_state_root_handle_for(parent_hash, parent_header, timestamp, state); + + PayloadBuilderResources::new(execution_cache, state_root_handle) } } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 6560ae225bc..2d077499b06 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -694,7 +694,7 @@ fn process_payload_attributes_shares_sparse_trie_during_validation_fallback() { let PayloadServiceCommand::BuildNewPayload(input, _, _) = command else { panic!("expected build new payload command") }; - assert!(input.state_root_handle.is_some()); + assert!(input.resources.state_root_handle().is_some()); } #[tokio::test] diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index 9e2f4ff33ef..a78f7967668 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -164,7 +164,8 @@ where input: BuildNewPayload, id: PayloadId, ) -> Result { - let parent_header = if input.parent_hash.is_zero() { + let BuildNewPayload { attributes, parent_hash, mut resources } = input; + let parent_header = if parent_hash.is_zero() { // Use latest header for genesis block case self.client .latest_header() @@ -173,16 +174,16 @@ where } else { // Fetch specific header by hash self.client - .sealed_header_by_hash(input.parent_hash) + .sealed_header_by_hash(parent_hash) .map_err(PayloadBuilderError::from)? - .ok_or_else(|| PayloadBuilderError::MissingParentHeader(input.parent_hash))? + .ok_or_else(|| PayloadBuilderError::MissingParentHeader(parent_hash))? }; let parent_hash = parent_header.hash(); let cached_reads = self.maybe_pre_cached(parent_hash); let parent_block_info = self.maybe_parent_block_info(parent_hash); - let config = PayloadConfig::new(Arc::new(parent_header), input.attributes, id) + let config = PayloadConfig::new(Arc::new(parent_header), attributes, id) .with_parent_block_info(parent_block_info); let until = self.job_deadline(config.attributes.timestamp()); @@ -197,8 +198,8 @@ where best_payload: PayloadState::Missing, pending_block: None, cached_reads, - execution_cache: input.cache, - state_root_handle: input.state_root_handle, + execution_cache: resources.take_execution_cache(), + state_root_handle: resources.take_state_root_handle(), payload_task_guard: self.payload_task_guard.clone(), metrics: Default::default(), builder: self.builder.clone(), diff --git a/crates/payload/builder/src/lib.rs b/crates/payload/builder/src/lib.rs index 7b1e4e27846..e5aeecc077a 100644 --- a/crates/payload/builder/src/lib.rs +++ b/crates/payload/builder/src/lib.rs @@ -126,8 +126,8 @@ pub use alloy_rpc_types::engine::PayloadId; pub use reth_payload_builder_primitives::PayloadBuilderError; pub use reth_payload_primitives::PayloadKind; pub use service::{ - BuildNewPayload, PayloadBuilderHandle, PayloadBuilderService, PayloadServiceCommand, - PayloadStore, + BuildNewPayload, PayloadBuilderHandle, PayloadBuilderLease, PayloadBuilderResources, + PayloadBuilderService, PayloadServiceCommand, PayloadStore, }; pub use traits::{KeepPayloadJobAlive, PayloadJob, PayloadJobGenerator}; diff --git a/crates/payload/builder/src/service.rs b/crates/payload/builder/src/service.rs index 0c18ac8a6d0..fc7c684ad45 100644 --- a/crates/payload/builder/src/service.rs +++ b/crates/payload/builder/src/service.rs @@ -32,7 +32,6 @@ use tokio_stream::wrappers::UnboundedReceiverStream; use tracing::{debug, debug_span, info, trace, warn, Span}; type PayloadFuture

= Pin> + Send>>; -type PayloadJobEntry = (Job, PayloadId, Span); type ResolvePayloadResult = (Option>, Option>); /// A communication channel to the [`PayloadBuilderService`] that can retrieve payloads. @@ -223,7 +222,7 @@ where /// All active payload jobs, each accompanied by its id and the caller's tracing span /// propagated across the channel so that poll and resolve work appears as children of the /// original Engine API request. - payload_jobs: Vec<(Gen::Job, PayloadId, Span)>, + payload_jobs: Vec>, /// Copy of the sender half, so new [`PayloadBuilderHandle`] can be created on demand. service_tx: mpsc::UnboundedSender>, /// Receiver half of the command channel. @@ -293,7 +292,7 @@ where /// Returns true if the given payload is currently being built. fn contains_payload(&self, id: PayloadId) -> bool { - self.payload_jobs.iter().any(|(_, job_id, _)| *job_id == id) + self.payload_jobs.iter().any(|entry| entry.id == id) } /// Returns the best payload for the given identifier that has been built so far. @@ -301,8 +300,8 @@ where let res = self .payload_jobs .iter() - .find(|(_, job_id, _)| *job_id == id) - .map(|(j, _, _)| j.best_payload().map(|p| p.into())); + .find(|entry| entry.id == id) + .map(|entry| entry.job.best_payload().map(|payload| payload.into())); if let Some(Ok(ref best)) = res { self.metrics.set_best_revenue(best.block().number(), f64::from(best.fees())); } @@ -329,14 +328,18 @@ where return (Some(Box::pin(core::future::ready(Ok(payload.clone())))), None); } - let Some(job) = self.payload_jobs.iter().position(|(_, job_id, _)| *job_id == id) else { + let Some(job) = self.payload_jobs.iter().position(|entry| entry.id == id) else { return (None, None) }; - let (fut, keep_alive) = self.payload_jobs[job].0.resolve_kind(kind); - let payload_timestamp = self.payload_jobs[job].0.payload_timestamp(); + let (fut, keep_alive) = self.payload_jobs[job].job.resolve_kind(kind); + let payload_timestamp = self.payload_jobs[job].job.payload_timestamp(); - let resolved_job = + let mut resolved_job = (keep_alive == KeepPayloadJobAlive::No).then(|| self.payload_jobs.swap_remove(job)); + let leases = resolved_job + .as_mut() + .map(|entry| std::mem::take(&mut entry.leases)) + .unwrap_or_default(); // Since the fees will not be known until the payload future is resolved / awaited, we wrap // the future in a new future that will update the metrics. @@ -345,6 +348,7 @@ where let cached_payload_tx = self.cached_payload_tx.clone(); let fut = async move { + let _leases = leases; let res = fut.await; resolved_metrics.resolve_duration_seconds.record(start.elapsed()); if let Ok(payload) = &res { @@ -376,8 +380,8 @@ where let timestamp = self .payload_jobs .iter() - .find(|(_, job_id, _)| *job_id == id) - .map(|(j, _, _)| j.payload_timestamp()); + .find(|entry| entry.id == id) + .map(|entry| entry.job.payload_timestamp()); if timestamp.is_none() { trace!(target: "payload_builder", %id, "no matching payload job found to get timestamp for"); @@ -411,10 +415,11 @@ where // requests // we don't care about the order of the jobs, so we can just swap_remove them for idx in (0..this.payload_jobs.len()).rev() { - let (mut job, id, job_span) = this.payload_jobs.swap_remove(idx); + let PayloadJobEntry { mut job, id, span, leases } = + this.payload_jobs.swap_remove(idx); let poll_result = { - let _entered = job_span.enter(); + let _entered = span.enter(); job.poll_unpin(cx) }; @@ -429,7 +434,7 @@ where this.metrics.set_active_jobs(this.payload_jobs.len()); } Poll::Pending => { - this.payload_jobs.push((job, id, job_span)); + this.payload_jobs.push(PayloadJobEntry { job, id, span, leases }); } } } @@ -440,7 +445,7 @@ where // drain all requests while let Poll::Ready(Some(cmd)) = this.command_rx.poll_next_unpin(cx) { match cmd { - PayloadServiceCommand::BuildNewPayload(input, job_span, tx) => { + PayloadServiceCommand::BuildNewPayload(mut input, job_span, tx) => { let id = input.payload_id(); let mut res = Ok(id); let parent = input.parent_hash; @@ -450,6 +455,7 @@ where } else { let start = Instant::now(); let attributes = input.attributes.clone(); + let leases = input.resources.take_leases(); let job_result = { let _entered = job_span.enter(); this.generator.new_payload_job(*input, id) @@ -461,7 +467,12 @@ where info!(target: "payload_builder", %id, %parent, "New payload job created"); this.metrics.inc_initiated_jobs(); new_job = true; - this.payload_jobs.push((job, id, job_span)); + this.payload_jobs.push(PayloadJobEntry { + job, + id, + span: job_span, + leases, + }); this.payload_events.send(Events::Attributes(attributes)).ok(); // Clear stale cached payload for this id so @@ -499,8 +510,8 @@ where let (payload_fut, resolved_job) = this.resolve(id, strategy); let _ = tx.send(payload_fut); - if let Some((_job, id, _job_span)) = resolved_job { - debug!(target: "payload_builder", %id, "terminated resolved job"); + if let Some(entry) = resolved_job { + debug!(target: "payload_builder", id = %entry.id, "terminated resolved job"); } } PayloadServiceCommand::Subscribe(tx) => { @@ -550,12 +561,8 @@ pub struct BuildNewPayload { pub attributes: T, /// The parent hash of the new payload pub parent_hash: B256, - /// Optional execution cache to use for the payload. - /// - /// Only provided if `--engine.share-execution-cache-with-payload-builder` is enabled. - pub cache: Option, - /// Optional handle to a background state-root task. - pub state_root_handle: Option, + /// Resources loaned to the payload builder for this job. + pub resources: PayloadBuilderResources, } impl BuildNewPayload { @@ -564,3 +571,131 @@ impl BuildNewPayload { self.attributes.payload_id(&self.parent_hash) } } + +/// Resources loaned to a payload builder job by the engine. +#[derive(Debug, Default)] +pub struct PayloadBuilderResources { + /// Optional execution cache to use for the payload. + /// + /// Only provided if `--engine.share-execution-cache-with-payload-builder` is enabled. + execution_cache: Option, + /// Optional handle to a background state-root task. + state_root_handle: Option, + /// Lifecycle leases retained by the service while the payload job is active. + leases: Vec, +} + +impl PayloadBuilderResources { + /// Creates a new payload builder resource bundle. + pub const fn new( + execution_cache: Option, + state_root_handle: Option, + ) -> Self { + Self { execution_cache, state_root_handle, leases: Vec::new() } + } + + /// Adds a lease that remains active for the lifetime of the payload job. + pub fn with_lease(mut self, lease: PayloadBuilderLease) -> Self { + self.leases.push(lease); + self + } + + /// Returns the loaned execution cache, if any. + pub const fn execution_cache(&self) -> Option<&SavedCache> { + self.execution_cache.as_ref() + } + + /// Takes the loaned execution cache, if any. + pub const fn take_execution_cache(&mut self) -> Option { + self.execution_cache.take() + } + + /// Returns the loaned state-root task handle, if any. + pub const fn state_root_handle(&self) -> Option<&PayloadStateRootHandle> { + self.state_root_handle.as_ref() + } + + /// Takes the loaned state-root task handle, if any. + pub const fn take_state_root_handle(&mut self) -> Option { + self.state_root_handle.take() + } + + /// Takes the lifecycle leases that the service must retain for this job. + fn take_leases(&mut self) -> Vec { + std::mem::take(&mut self.leases) + } +} + +/// Keeps a loaned resource active for the lifetime of a payload job. +pub struct PayloadBuilderLease { + _lease: Box, +} + +impl PayloadBuilderLease { + /// Wraps a lease that releases its resource when dropped. + pub fn new(lease: impl Send + 'static) -> Self { + Self { _lease: Box::new(lease) } + } +} + +impl std::fmt::Debug for PayloadBuilderLease { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("PayloadBuilderLease").finish_non_exhaustive() + } +} + +/// An active payload job and its service metadata. +#[derive(Debug)] +struct PayloadJobEntry { + job: Job, + id: PayloadId, + span: Span, + leases: Vec, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::test_payload_service; + use alloy_primitives::Address; + use reth_ethereum_engine_primitives::{EthEngineTypes, EthPayloadAttributes}; + use std::sync::atomic::{AtomicBool, Ordering}; + + struct DropProbe(Arc); + + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.store(true, Ordering::Release); + } + } + + #[test] + fn payload_builder_lease_is_held_until_resolve_finishes() { + tokio::runtime::Builder::new_current_thread().build().unwrap().block_on(async { + let (service, handle) = test_payload_service::(); + let service = tokio::spawn(service); + let dropped = Arc::new(AtomicBool::new(false)); + let lease = PayloadBuilderLease::new(DropProbe(Arc::clone(&dropped))); + let input = BuildNewPayload { + attributes: EthPayloadAttributes { + timestamp: 1, + prev_randao: B256::ZERO, + suggested_fee_recipient: Address::ZERO, + withdrawals: None, + parent_beacon_block_root: None, + slot_number: None, + target_gas_limit: None, + }, + parent_hash: B256::ZERO, + resources: PayloadBuilderResources::default().with_lease(lease), + }; + + let id = handle.send_new_payload(input).await.unwrap().unwrap(); + assert!(!dropped.load(Ordering::Acquire)); + + handle.resolve_kind(id, PayloadKind::Earliest).await.unwrap().unwrap(); + assert!(dropped.load(Ordering::Acquire)); + service.abort(); + }); + } +} From 1cd853d601f88a40dba1e2d901e34551d271350c Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Fri, 17 Jul 2026 21:05:58 +0200 Subject: [PATCH 015/148] ci(bench): RPC replay mode (#26418) --- .github/scripts/bench-reth-summary.py | 12 +- .github/scripts/bench-txgen-extract.sh | 66 ++++++++--- .../scripts/bench-txgen-report-to-reth-csv.py | 20 +++- .github/scripts/bench-txgen-run.sh | 105 +++++++++++++----- .github/workflows/bench.yml | 86 ++++++++++++-- 5 files changed, 226 insertions(+), 63 deletions(-) diff --git a/.github/scripts/bench-reth-summary.py b/.github/scripts/bench-reth-summary.py index 9b942a183af..7d1ad07a2f4 100755 --- a/.github/scripts/bench-reth-summary.py +++ b/.github/scripts/bench-reth-summary.py @@ -1659,7 +1659,9 @@ def generate_comparison_table( big_blocks: bool = False, warmup_blocks: str | None = None, wait_time: str | None = None, + block_time: str | None = None, bal_mode: str | None = None, + mode: str = "engine", run_pairs: int | None = None, ) -> str: """Generate a markdown comparison table between baseline and feature.""" @@ -1715,13 +1717,15 @@ def pct(base: float, feat: float) -> float: f"| Persist Wait | {fmt_ms(run1['mean_persist_ms'])} | {fmt_ms(run2['mean_persist_ms'])} | {change_str(persist_pct, persist_ci_pct, persist_floor, lower_is_better=True, informational=persist_informational)} |", "", ] - meta_parts = [f"{n} {'big blocks' if big_blocks else 'blocks'}"] + meta_parts = [f"{n} {'big blocks' if big_blocks else 'blocks'}", f"mode: {mode}"] if warmup_blocks: meta_parts.append(f"{warmup_blocks} warmup") if run_pairs: meta_parts.append(f"{run_pairs} run pairs") if wait_time: meta_parts.append(f"wait time: {wait_time}") + if block_time: + meta_parts.append(f"block time: {block_time}") display_mode = display_bal_mode(bal_mode) if big_blocks and display_mode: meta_parts.append(f"BAL: {display_mode}") @@ -1969,7 +1973,9 @@ def main(): parser.add_argument("--big-blocks", action="store_true", default=False, help="Big blocks mode") parser.add_argument("--warmup-blocks", default=None, help="Number of warmup blocks") parser.add_argument("--wait-time", default=None, help="Wait time interval used between blocks") + parser.add_argument("--block-time", default=None, help="RPC mode local block interval") parser.add_argument("--bal-mode", default=None, help="BAL mode (true, feature, baseline)") + parser.add_argument("--mode", choices=("engine", "rpc"), default="engine", help="Benchmark execution mode") parser.add_argument("--benchmark-id", default=os.environ.get("BENCH_ID"), help="Benchmark ID used for OTLP labels") parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL for this benchmark run") parser.add_argument("--logs-url", default=None, help="Grafana Explore URL for benchmark logs") @@ -2033,7 +2039,9 @@ def main(): big_blocks=args.big_blocks, warmup_blocks=args.warmup_blocks, wait_time=args.wait_time, + block_time=args.block_time, bal_mode=bal_mode, + mode=args.mode, run_pairs=args.run_pairs, ) print( @@ -2084,7 +2092,9 @@ def main(): "warmup_blocks": args.warmup_blocks, "run_pairs": args.run_pairs, "wait_time": args.wait_time, + "block_time": args.block_time, "bal_mode": bal_mode, + "mode": args.mode, "baseline": { "name": baseline_name, "ref": baseline_ref, diff --git a/.github/scripts/bench-txgen-extract.sh b/.github/scripts/bench-txgen-extract.sh index 39131424021..a513dcfd1d4 100755 --- a/.github/scripts/bench-txgen-extract.sh +++ b/.github/scripts/bench-txgen-extract.sh @@ -1,11 +1,11 @@ #!/usr/bin/env bash # -# Pre-extracts txgen payloads (normal or big blocks) so they can be reused +# Pre-extracts txgen payloads or transactions so they can be reused # across multiple benchmark runs instead of re-fetching from the remote RPC # every time. # # Starts a throwaway reth node to discover the snapshot's chain tip, extracts -# payloads from BENCH_RPC_URL, then stops the node and recovers the snapshot. +# data from BENCH_RPC_URL, then stops the node and recovers the snapshot. # # Usage: bench-txgen-extract.sh # @@ -13,15 +13,17 @@ # /all-blocks.ndjson (or all-big-blocks.ndjson) # /warmup-blocks.ndjson # /benchmark-blocks.ndjson +# In RPC mode, the corresponding names end in `-transactions.ndjson`. # # Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS -# Optional env: BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL +# Optional env: BENCH_EXECUTION_MODE, BENCH_BIG_BLOCKS, BENCH_BIG_BLOCKS_TARGET_GAS, BENCH_BAL set -euxo pipefail BINARY="$1" OUTPUT_DIR="$2" BIG_BLOCKS="${BENCH_BIG_BLOCKS:-false}" +EXECUTION_MODE="${BENCH_EXECUTION_MODE:-engine}" BAL_MODE="${BENCH_BAL:-false}" INCLUDE_BAL=false if [ "$BAL_MODE" != "false" ] && [ -n "$BAL_MODE" ]; then @@ -123,19 +125,44 @@ fi ALL_BLOCKS="$OUTPUT_DIR/all-blocks.ndjson" WARMUP_FILE="$OUTPUT_DIR/warmup-blocks.ndjson" BENCHMARK_FILE="$OUTPUT_DIR/benchmark-blocks.ndjson" - -if [ "$BIG_BLOCKS" = "true" ]; then - ALL_BLOCKS="$OUTPUT_DIR/all-big-blocks.ndjson" - WARMUP_FILE="$OUTPUT_DIR/warmup-big-blocks.ndjson" - BENCHMARK_FILE="$OUTPUT_DIR/measured-big-blocks.ndjson" -fi - EXTRACT_FROM=$(( HEAD_DEC + 1 )) TXGEN_EXTRACT_ARGS=() if [ "$INCLUDE_BAL" = "true" ]; then TXGEN_EXTRACT_ARGS+=(--bal) fi -if [ "$BIG_BLOCKS" = "true" ]; then + +if [ "$EXECUTION_MODE" = "rpc" ]; then + ALL_BLOCKS="$OUTPUT_DIR/all-transactions.ndjson" + WARMUP_FILE="$OUTPUT_DIR/warmup-transactions.ndjson" + BENCHMARK_FILE="$OUTPUT_DIR/benchmark-transactions.ndjson" + if [ "$BIG_BLOCKS" = "true" ] || [ "$INCLUDE_BAL" = "true" ]; then + echo "::error::RPC mode does not support big blocks or BAL" + exit 1 + fi + + EXTRACT_TO=$(( HEAD_DEC + TOTAL )) + echo "Extracting transactions from blocks ${EXTRACT_FROM}..${EXTRACT_TO} for RPC benchmark (${WARMUP} warmup blocks, ${BLOCKS} measured blocks)" + if [ "$WARMUP" -gt 0 ] 2>/dev/null; then + "$TXGEN_ETHEREUM" extract \ + --rpc "$BENCH_RPC_URL" \ + --from "$EXTRACT_FROM" \ + --to "$(( HEAD_DEC + WARMUP ))" \ + --format transactions \ + -o "$WARMUP_FILE" + else + : > "$WARMUP_FILE" + fi + "$TXGEN_ETHEREUM" extract \ + --rpc "$BENCH_RPC_URL" \ + --from "$(( HEAD_DEC + WARMUP + 1 ))" \ + --to "$EXTRACT_TO" \ + --format transactions \ + -o "$BENCHMARK_FILE" + cat "$WARMUP_FILE" "$BENCHMARK_FILE" > "$ALL_BLOCKS" +elif [ "$BIG_BLOCKS" = "true" ]; then + ALL_BLOCKS="$OUTPUT_DIR/all-big-blocks.ndjson" + WARMUP_FILE="$OUTPUT_DIR/warmup-big-blocks.ndjson" + BENCHMARK_FILE="$OUTPUT_DIR/measured-big-blocks.ndjson" echo "Extracting ${TOTAL} big blocks from ${EXTRACT_FROM} for txgen benchmark (${WARMUP} warmup, ${BLOCKS} measured, bal=${INCLUDE_BAL})" "$TXGEN_ETHEREUM" extract-big-blocks \ --rpc "$BENCH_RPC_URL" \ @@ -155,13 +182,16 @@ else -o "$ALL_BLOCKS" fi -# Split into warmup and measured files -if [ "$WARMUP" -gt 0 ] 2>/dev/null; then - head -n "$WARMUP" "$ALL_BLOCKS" > "$WARMUP_FILE" -else - : > "$WARMUP_FILE" +# Block output has one line per source block. Transaction output is extracted +# separately above because it has one line per transaction. +if [ "$EXECUTION_MODE" = "engine" ]; then + if [ "$WARMUP" -gt 0 ] 2>/dev/null; then + head -n "$WARMUP" "$ALL_BLOCKS" > "$WARMUP_FILE" + else + : > "$WARMUP_FILE" + fi + awk -v warmup="$WARMUP" 'NR > warmup { print }' "$ALL_BLOCKS" > "$BENCHMARK_FILE" fi -awk -v warmup="$WARMUP" 'NR > warmup { print }' "$ALL_BLOCKS" > "$BENCHMARK_FILE" if [ "$INCLUDE_BAL" = "true" ] && [ "$BAL_MODE" != "true" ]; then echo "Writing no-BAL payload variants for selective BAL mode (${BAL_MODE})" @@ -170,4 +200,4 @@ if [ "$INCLUDE_BAL" = "true" ] && [ "$BAL_MODE" != "true" ]; then done fi -echo "Extraction complete: $(wc -l < "$ALL_BLOCKS") payloads in ${OUTPUT_DIR}" +echo "Extraction complete: $(wc -l < "$ALL_BLOCKS") ${EXECUTION_MODE} records in ${OUTPUT_DIR}" diff --git a/.github/scripts/bench-txgen-report-to-reth-csv.py b/.github/scripts/bench-txgen-report-to-reth-csv.py index 6501abe617a..a9847014e45 100755 --- a/.github/scripts/bench-txgen-report-to-reth-csv.py +++ b/.github/scripts/bench-txgen-report-to-reth-csv.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Convert txgen `bench send-blocks` JSON into legacy benchmark CSVs. +"""Convert txgen `bench send-blocks` or `bench send` JSON into legacy benchmark CSVs. The benchmark rendering pipeline still consumes `combined_latency.csv` and `total_gas.csv`. This adapter lets the txgen runner reuse the existing @@ -18,12 +18,17 @@ def opt_int(value, default=None): return int(value) -def block_latency_us(block: dict) -> tuple[int, int, int]: +def block_latency_us(block: dict, fallback_block_time_ms: int = 0) -> tuple[int, int, int]: # txgen currently records server newPayload latency in microseconds but # client-side forkchoiceUpdated latency in milliseconds. new_payload_us = opt_int(block.get("new_payload_server_latency_us")) if new_payload_us is None: - new_payload_us = opt_int(block.get("new_payload_ms"), 0) * 1000 + new_payload_ms = opt_int(block.get("new_payload_ms")) + if new_payload_ms is None: + # `bench send` has no Engine API timings. Use locally mined block + # time so the existing gas/latency summaries remain meaningful. + new_payload_ms = opt_int(block.get("block_time_ms"), fallback_block_time_ms) + new_payload_us = new_payload_ms * 1000 fcu_us = opt_int(block.get("forkchoice_updated_ms"), 0) * 1000 return new_payload_us, fcu_us, new_payload_us + fcu_us @@ -45,6 +50,11 @@ def main() -> None: if not blocks: raise SystemExit(f"txgen report {report_path} does not contain any blocks") + fallback_block_time_ms = next( + (int(block["block_time_ms"]) for block in blocks if block.get("block_time_ms") is not None), + 0, + ) + combined_path = output_dir / "combined_latency.csv" with combined_path.open("w", newline="") as f: writer = csv.DictWriter( @@ -64,7 +74,7 @@ def main() -> None: ) writer.writeheader() for block in blocks: - new_payload_us, fcu_us, total_us = block_latency_us(block) + new_payload_us, fcu_us, total_us = block_latency_us(block, fallback_block_time_ms) writer.writerow( { "block_number": block["number"], @@ -89,7 +99,7 @@ def main() -> None: ) writer.writeheader() for block in blocks: - _, _, total_us = block_latency_us(block) + _, _, total_us = block_latency_us(block, fallback_block_time_ms) elapsed_us += total_us writer.writerow( { diff --git a/.github/scripts/bench-txgen-run.sh b/.github/scripts/bench-txgen-run.sh index 5631ff70204..b709cf7c544 100755 --- a/.github/scripts/bench-txgen-run.sh +++ b/.github/scripts/bench-txgen-run.sh @@ -1,14 +1,15 @@ #!/usr/bin/env bash # -# Runs a single txgen-backed Engine API benchmark cycle: -# mount snapshot → start node → extract source blocks → warmup → send-blocks → +# Runs a single txgen-backed benchmark cycle: +# mount snapshot → start node → extract source blocks → warmup → replay → # convert txgen JSON report into legacy benchmark CSVs. # # Usage: bench-txgen-run.sh

(P); + +impl

Source

{ + /// Creates a new txpool prewarm source. + pub(crate) const fn new(pool: P) -> Self { + Self(pool) + } +} + +impl PrewarmSource for Source

+where + N: NodePrimitives, + P: TransactionPool>> + + Clone + + Send + + Sync + + Debug + + 'static, +{ + fn best_transactions(&self, parent_hash: B256) -> Option> { + let block_info = self.0.block_info(); + if block_info.last_seen_block_hash != parent_hash { + return None + } + + let mut best = self.0.best_transactions_with_attributes(BestTransactionsAttributes::new( + block_info.pending_basefee, + block_info.pending_blob_fee.map(|fee| u64::try_from(fee).unwrap_or(u64::MAX)), + )); + best.allow_updates_out_of_order(); + best.skip_blobs(); + + Some(Box::new(best.map(|transaction| Transaction { + hash: *transaction.hash(), + sender: transaction.sender(), + transaction: transaction.transaction.clone_into_consensus(), + }))) + } +} diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index 3d118d04836..b6e0a60c2cb 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -31,6 +31,7 @@ pub struct DefaultEngineValues { invalid_header_hit_eviction_threshold: u8, state_cache_disabled: bool, prewarming_disabled: bool, + txpool_prewarming_enabled: bool, state_provider_metrics: bool, cross_block_cache_size: usize, state_root_task_compare_updates: bool, @@ -102,6 +103,12 @@ impl DefaultEngineValues { self } + /// Set whether to enable txpool prewarming by default + pub const fn with_txpool_prewarming_enabled(mut self, v: bool) -> Self { + self.txpool_prewarming_enabled = v; + self + } + /// Set whether to enable state provider metrics by default pub const fn with_state_provider_metrics(mut self, v: bool) -> Self { self.state_provider_metrics = v; @@ -247,6 +254,7 @@ impl Default for DefaultEngineValues { invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, state_cache_disabled: false, prewarming_disabled: false, + txpool_prewarming_enabled: false, state_provider_metrics: false, cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, state_root_task_compare_updates: false, @@ -337,6 +345,14 @@ pub struct EngineArgs { #[arg(long = "engine.disable-prewarming", alias = "engine.disable-caching-and-prewarming", default_value_t = DefaultEngineValues::get_global().prewarming_disabled)] pub prewarming_disabled: bool, + /// Enable best-effort txpool transaction prewarming between payloads. + #[arg( + long = "engine.txpool-prewarming", + default_value_t = DefaultEngineValues::get_global().txpool_prewarming_enabled, + conflicts_with = "state_cache_disabled" + )] + pub txpool_prewarming_enabled: bool, + /// CAUTION: This CLI flag has no effect anymore. The parallel sparse trie is always enabled. #[deprecated] #[arg(long = "engine.parallel-sparse-trie", default_value = "true", hide = true)] @@ -534,6 +550,7 @@ impl Default for EngineArgs { invalid_header_hit_eviction_threshold, state_cache_disabled, prewarming_disabled, + txpool_prewarming_enabled, state_provider_metrics, cross_block_cache_size, state_root_task_compare_updates, @@ -567,6 +584,7 @@ impl Default for EngineArgs { caching_and_prewarming_enabled: true, state_cache_disabled, prewarming_disabled, + txpool_prewarming_enabled, parallel_sparse_trie_enabled: true, parallel_sparse_trie_disabled: false, state_provider_metrics, @@ -652,6 +670,7 @@ impl EngineArgs { .with_invalid_header_hit_eviction_threshold(self.invalid_header_hit_eviction_threshold) .without_state_cache(self.state_cache_disabled) .without_prewarming(self.prewarming_disabled) + .with_txpool_prewarming(self.txpool_prewarming_enabled) .with_state_provider_metrics(self.state_provider_metrics) .with_always_compare_trie_updates(self.state_root_task_compare_updates) .with_cross_block_cache_size(self.cross_block_cache_size * 1024 * 1024) @@ -709,6 +728,30 @@ mod tests { ); } + #[test] + fn txpool_prewarming_is_disabled_by_default_and_can_be_enabled() { + let args = CommandParser::::parse_from(["reth"]).args; + assert!(!args.txpool_prewarming_enabled); + assert!(!args.tree_config().txpool_prewarming()); + + let args = + CommandParser::::parse_from(["reth", "--engine.txpool-prewarming"]).args; + assert!(args.txpool_prewarming_enabled); + assert!(args.tree_config().txpool_prewarming()); + } + + #[test] + fn txpool_prewarming_conflicts_with_disabled_state_cache() { + let Err(error) = CommandParser::::try_parse_from([ + "reth", + "--engine.txpool-prewarming", + "--engine.disable-state-cache", + ]) else { + panic!("conflicting flags must be rejected") + }; + assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); + } + #[test] fn default_backpressure_threshold_uses_parsed_persistence_args() { let args = CommandParser::::parse_from([ @@ -771,6 +814,8 @@ mod tests { caching_and_prewarming_enabled: true, state_cache_disabled: true, prewarming_disabled: true, + // conflicts with --engine.disable-state-cache, covered by its own test below + txpool_prewarming_enabled: false, parallel_sparse_trie_enabled: true, parallel_sparse_trie_disabled: false, state_provider_metrics: true, diff --git a/crates/node/events/src/node.rs b/crates/node/events/src/node.rs index 7223026273f..7ee9ef5660f 100644 --- a/crates/node/events/src/node.rs +++ b/crates/node/events/src/node.rs @@ -337,6 +337,15 @@ impl NodeState { cache.code.hits = stats.code_cache_hits, cache.code.misses = stats.code_cache_misses, cache.code.hit_rate = format!("{:.2}", hit_rate(stats.code_cache_hits, stats.code_cache_misses)), + cache.txpool_snapshot.account.hits = stats.txpool_snapshot_account_hits, + cache.txpool_snapshot.account.misses = stats.txpool_snapshot_account_misses, + cache.txpool_snapshot.account.hit_rate = format!("{:.2}", hit_rate(stats.txpool_snapshot_account_hits, stats.txpool_snapshot_account_misses)), + cache.txpool_snapshot.storage.hits = stats.txpool_snapshot_storage_hits, + cache.txpool_snapshot.storage.misses = stats.txpool_snapshot_storage_misses, + cache.txpool_snapshot.storage.hit_rate = format!("{:.2}", hit_rate(stats.txpool_snapshot_storage_hits, stats.txpool_snapshot_storage_misses)), + cache.txpool_snapshot.code.hits = stats.txpool_snapshot_code_hits, + cache.txpool_snapshot.code.misses = stats.txpool_snapshot_code_misses, + cache.txpool_snapshot.code.hit_rate = format!("{:.2}", hit_rate(stats.txpool_snapshot_code_hits, stats.txpool_snapshot_code_misses)), ); } } diff --git a/crates/transaction-pool/src/pool/best.rs b/crates/transaction-pool/src/pool/best.rs index 6d6b46cdf8e..e3e214dd9d8 100644 --- a/crates/transaction-pool/src/pool/best.rs +++ b/crates/transaction-pool/src/pool/best.rs @@ -41,6 +41,10 @@ impl crate::traits::BestTransactions for BestTransaction self.best.no_updates() } + fn allow_updates_out_of_order(&mut self) { + self.best.allow_updates_out_of_order() + } + fn skip_blobs(&mut self) { self.set_skip_blobs(true) } @@ -111,6 +115,8 @@ pub struct BestTransactions { pub(crate) last_priority: Option>, /// Flag to control whether to skip blob transactions (EIP4844). pub(crate) skip_blobs: bool, + /// Whether live updates can be yielded after a lower-priority transaction. + pub(crate) allow_updates_out_of_order: bool, } impl BestTransactions { @@ -136,7 +142,8 @@ impl BestTransactions { loop { match self.new_transaction_receiver.as_mut()?.try_recv() { Ok(tx) => { - if let Some(last_priority) = &self.last_priority && + if !self.allow_updates_out_of_order && + let Some(last_priority) = &self.last_priority && &tx.priority > last_priority { // we skip transactions if we already yielded a transaction with lower @@ -278,6 +285,10 @@ impl crate::traits::BestTransactions for BestTransaction self.last_priority.take(); } + fn allow_updates_out_of_order(&mut self) { + self.allow_updates_out_of_order = true; + } + fn skip_blobs(&mut self) { self.set_skip_blobs(true); } @@ -355,6 +366,10 @@ where self.best.no_updates() } + fn allow_updates_out_of_order(&mut self) { + self.best.allow_updates_out_of_order() + } + fn skip_blobs(&mut self) { self.set_skip_blobs(true) } @@ -454,6 +469,10 @@ where self.inner.no_updates() } + fn allow_updates_out_of_order(&mut self) { + self.inner.allow_updates_out_of_order() + } + fn set_skip_blobs(&mut self, skip_blobs: bool) { if skip_blobs { self.buffer.retain(|tx| !tx.transaction.is_eip4844()) @@ -1107,6 +1126,23 @@ mod tests { assert!(best.new_transaction_receiver.is_none()); } + #[test] + fn test_best_transactions_yields_updates_after_empty() { + let mut pool = PendingPool::new(MockOrdering::default()); + let mut best = pool.best(); + best.allow_updates_out_of_order(); + + assert!(best.next().is_none()); + + let mut f = MockTransactionFactory::default(); + let tx = MockTransaction::eip1559().rng_hash(); + let valid_tx = Arc::new(f.validated(tx)); + let expected_hash = *valid_tx.hash(); + pool.add_transaction(valid_tx, 0); + + assert_eq!(*best.next().expect("new transaction should be yielded").hash(), expected_hash); + } + #[test] fn test_best_update_transaction_priority() { let mut pool = PendingPool::new(MockOrdering::default()); diff --git a/crates/transaction-pool/src/pool/pending.rs b/crates/transaction-pool/src/pool/pending.rs index 57a70c90c91..7ca27ad9b66 100644 --- a/crates/transaction-pool/src/pool/pending.rs +++ b/crates/transaction-pool/src/pool/pending.rs @@ -109,6 +109,7 @@ impl PendingPool { new_transaction_receiver: Some(self.new_transaction_notifier.subscribe()), last_priority: None, skip_blobs: false, + allow_updates_out_of_order: false, } } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 654ef7e5a87..877e1e6e6f1 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1119,6 +1119,14 @@ pub trait BestTransactions: Iterator + Send { /// listen to pool updates. fn no_updates(&mut self); + /// Allows newly received transactions to be yielded even if their priority is higher than a + /// transaction that was already yielded. + /// + /// This is useful for long-lived consumers that prefer seeing every update over preserving + /// decreasing priority order. The default implementation leaves the iterator's ordering + /// behavior unchanged. Implementations must still preserve transaction dependency ordering. + fn allow_updates_out_of_order(&mut self) {} + /// Convenience function for [`Self::no_updates`] that returns the iterator again. fn without_updates(mut self) -> Self where @@ -1180,6 +1188,10 @@ where (**self).no_updates(); } + fn allow_updates_out_of_order(&mut self) { + (**self).allow_updates_out_of_order(); + } + fn skip_blobs(&mut self) { (**self).skip_blobs(); } diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 3f6a0593a32..eaec84883dd 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1011,6 +1011,9 @@ Engine: --engine.disable-prewarming Disable parallel prewarming + --engine.txpool-prewarming + Enable best-effort txpool transaction prewarming between payloads + --engine.state-provider-metrics Enable state provider latency metrics. This allows the engine to collect and report stats about how long state provider calls took during execution, but this does introduce slight overhead to state provider calls From f3a9130f3627fd3cc050fd27c1199aa63a04e594 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Fri, 24 Jul 2026 13:20:01 +0200 Subject: [PATCH 049/148] feat: enable txpool prewarm on nightly (#26493) --- .github/workflows/docker.yml | 7 ++++++- Dockerfile.depot | 4 ++++ crates/node/core/src/args/engine.rs | 25 +++++++++++++++---------- docs/vocs/docs/pages/cli/reth/node.mdx | 2 ++ 4 files changed, 27 insertions(+), 11 deletions(-) diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 06d501f6415..7d8264745e0 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -91,7 +91,12 @@ jobs: elif [[ "${EVENT_NAME}" == "schedule" ]] || [[ "${BUILD_TYPE}" == "nightly" ]]; then echo "targets=nightly" >> "$GITHUB_OUTPUT" echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT" - echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT" + { + echo "ethereum_set<> "$GITHUB_OUTPUT" else # git-sha build diff --git a/Dockerfile.depot b/Dockerfile.depot index 0a8e4fca698..311ee8e0621 100644 --- a/Dockerfile.depot +++ b/Dockerfile.depot @@ -74,6 +74,10 @@ WORKDIR /app # Binary name for entrypoint ARG BINARY=reth +# Runtime defaults +ARG RETH_ENGINE_TXPOOL_PREWARMING=false +ENV RETH_ENGINE_TXPOOL_PREWARMING=$RETH_ENGINE_TXPOOL_PREWARMING + # Install runtime dependencies RUN apt-get update && \ apt-get install -y --no-install-recommends ca-certificates && \ diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index b6e0a60c2cb..66d7e19b5ee 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -348,8 +348,8 @@ pub struct EngineArgs { /// Enable best-effort txpool transaction prewarming between payloads. #[arg( long = "engine.txpool-prewarming", - default_value_t = DefaultEngineValues::get_global().txpool_prewarming_enabled, - conflicts_with = "state_cache_disabled" + env = "RETH_ENGINE_TXPOOL_PREWARMING", + default_value_t = DefaultEngineValues::get_global().txpool_prewarming_enabled )] pub txpool_prewarming_enabled: bool, @@ -650,6 +650,10 @@ impl EngineArgs { memory_block_buffer_target, self.persistence_threshold, ); + ensure!( + !self.state_cache_disabled || !self.txpool_prewarming_enabled, + "--engine.txpool-prewarming conflicts with --engine.disable-state-cache" + ); ensure!( self.bal_parallel_execution_disabled || !self.bal_parallel_state_root_disabled, "--engine.disable-bal-parallel-state-root requires --engine.disable-bal-parallel-execution because BAL parallel execution depends on BAL prewarm state-root updates" @@ -741,15 +745,16 @@ mod tests { } #[test] - fn txpool_prewarming_conflicts_with_disabled_state_cache() { - let Err(error) = CommandParser::::try_parse_from([ - "reth", - "--engine.txpool-prewarming", - "--engine.disable-state-cache", - ]) else { - panic!("conflicting flags must be rejected") + fn validate_rejects_txpool_prewarming_with_disabled_state_cache() { + let args = EngineArgs { + state_cache_disabled: true, + txpool_prewarming_enabled: true, + ..EngineArgs::default() }; - assert_eq!(error.kind(), clap::error::ErrorKind::ArgumentConflict); + + let err = args.validate().unwrap_err().to_string(); + assert!(err.contains("engine.txpool-prewarming")); + assert!(err.contains("engine.disable-state-cache")); } #[test] diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index eaec84883dd..b63deba860b 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1014,6 +1014,8 @@ Engine: --engine.txpool-prewarming Enable best-effort txpool transaction prewarming between payloads + [env: RETH_ENGINE_TXPOOL_PREWARMING=] + --engine.state-provider-metrics Enable state provider latency metrics. This allows the engine to collect and report stats about how long state provider calls took during execution, but this does introduce slight overhead to state provider calls From 84f4c989cb29c14af6a7bdcc0a97e4844f1cd3da Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 24 Jul 2026 16:00:45 +0200 Subject: [PATCH 050/148] feat(trie): prune sparse trie nodes by epoch (#26485) --- .github/workflows/e2e.yml | 1 + Cargo.lock | 1 - crates/engine/tree/Cargo.toml | 1 - crates/engine/tree/src/tree/mod.rs | 2 +- .../engine/tree/src/tree/payload_validator.rs | 1 + .../tree/src/tree/state_root_strategy/mod.rs | 293 +++---------- .../tree/state_root_strategy/sparse_trie.rs | 27 +- crates/trie/sparse/src/arena/cursor.rs | 5 +- crates/trie/sparse/src/arena/mod.rs | 385 ++++++++--------- crates/trie/sparse/src/arena/nodes.rs | 32 +- crates/trie/sparse/src/state.rs | 185 ++++---- crates/trie/sparse/src/traits.rs | 61 ++- crates/trie/sparse/src/trie.rs | 12 +- crates/trie/sparse/tests/suite/lifecycle.rs | 149 ++----- crates/trie/sparse/tests/suite/main.rs | 23 +- crates/trie/sparse/tests/suite/prune.rs | 397 ++++++------------ .../trie/sparse/tests/suite/reveal_nodes.rs | 28 +- crates/trie/sparse/tests/suite/root.rs | 18 +- crates/trie/sparse/tests/suite/set_root.rs | 14 +- crates/trie/sparse/tests/suite/size_hint.rs | 52 --- .../trie/sparse/tests/suite/take_updates.rs | 20 +- .../trie/sparse/tests/suite/update_leaves.rs | 127 +++--- crates/trie/sparse/tests/suite/wipe_clear.rs | 18 +- crates/trie/trie/src/witness.rs | 4 +- 24 files changed, 723 insertions(+), 1133 deletions(-) delete mode 100644 crates/trie/sparse/tests/suite/size_hint.rs diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index d64335575fa..5a4d0686233 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -41,6 +41,7 @@ jobs: tool: nextest - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: + cache-targets: false cache-on-failure: true - name: Run e2e tests run: | diff --git a/Cargo.lock b/Cargo.lock index 8fd222d46c7..f75716d4972 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8336,7 +8336,6 @@ dependencies = [ "eyre", "futures", "indexmap 2.14.0", - "itertools 0.14.0", "metrics", "metrics-util", "moka", diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index 5dceb9476cc..95cb5dbc21d 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -61,7 +61,6 @@ reth-metrics = { workspace = true, features = ["common"] } # misc indexmap.workspace = true -itertools.workspace = true schnellru.workspace = true rayon.workspace = true tracing.workspace = true diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index f67a8732897..f5b0b840365 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -199,7 +199,7 @@ impl EngineApiTreeState { /// /// `None` means no prune request is pending. `Some(Vec::new())` means a prune was requested, /// but no in-memory parent-chain blocks were found for the parent hash; the sparse trie task - /// should still prune using the current block's hashed post state. + /// should still prune nodes cached before the current block's epoch. pub fn take_sparse_trie_prune_blocks( &mut self, parent_hash: B256, diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 4b524c60fa5..a31aa7f6f00 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -619,6 +619,7 @@ where &self.runtime, &self.state_trie_overlays, &env, + &parent_block, provider_builder.clone(), overlay_factory, &self.config, diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 1baf7452929..f69994e6a8a 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -43,7 +43,7 @@ //! //! Returning empty trie updates in the outcome means the trie tables are no longer maintained: //! `eth_getProof` and anything else that reads the stored trie will not work for new blocks. -//! Sparse-trie cache pruning derives retained paths from each block's hashed post state. +//! Sparse-trie cache pruning uses node epochs to retain the in-memory block range. mod sparse_trie; @@ -52,15 +52,13 @@ use crate::tree::{ metrics::BlockValidationMetrics, EngineApiTreeState, ExecutionEnv, StateProviderBuilder, TreeConfig, }; -use alloy_primitives::{map::B256Map, B256, U256}; +use alloy_primitives::B256; use crossbeam_channel::Receiver as CrossbeamReceiver; -use itertools::Itertools; -use rayon::{join, prelude::*}; use reth_chain_state::{ExecutedBlock, PreservedSparseTrie, StateTrieOverlayManager}; use reth_errors::ProviderResult; use reth_evm::{ConfigureEvm, OnStateHook}; use reth_primitives_traits::{ - AlloyBlockHeader, FastInstant as Instant, NodePrimitives, RecoveredBlock, + AlloyBlockHeader, FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedHeader, }; use reth_provider::{ providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader, @@ -69,10 +67,7 @@ use reth_provider::{ }; use reth_tasks::utils::increase_thread_priority; use reth_trie::{ - hashed_cursor::HashedCursorFactory, - prefix_set::{PrefixSet, TriePrefixSets}, - trie_cursor::TrieCursorFactory, - updates::TrieUpdates, + hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, }; use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofWorkerHandle}; @@ -86,7 +81,9 @@ pub use reth_trie_parallel::{ }; #[cfg(feature = "trie-debug")] use reth_trie_sparse::debug_recorder::TrieDebugRecorder; -use reth_trie_sparse::{ArenaParallelSparseTrie, RevealableSparseTrie, SparseStateTrie}; +use reth_trie_sparse::{ + ArenaParallelSparseTrie, RevealableSparseTrie, SparseStateTrie, TrieNodeEpoch, +}; use std::{ fmt, sync::{ @@ -245,6 +242,7 @@ where executor: &'a reth_tasks::Runtime, state_trie_overlays: &'a StateTrieOverlayManager, env: &'a ExecutionEnv, + parent_header: &'a SealedHeader, provider_builder: StateProviderBuilder, overlay_factory: OverlayStateProviderFactory, config: &'a TreeConfig, @@ -276,6 +274,7 @@ where executor: &'a reth_tasks::Runtime, state_trie_overlays: &'a StateTrieOverlayManager, env: &'a ExecutionEnv, + parent_header: &'a SealedHeader, provider_builder: StateProviderBuilder, overlay_factory: OverlayStateProviderFactory, config: &'a TreeConfig, @@ -286,6 +285,7 @@ where executor, state_trie_overlays, env, + parent_header, provider_builder, overlay_factory, config, @@ -299,6 +299,11 @@ where self.env } + /// Returns the sealed parent block header. + pub const fn parent_header(&self) -> &SealedHeader { + self.parent_header + } + /// Returns the task runtime used by state-root work. pub const fn executor(&self) -> &reth_tasks::Runtime { self.executor @@ -517,8 +522,7 @@ impl DefaultStateRootStrategy { + 'static, { let StateRootTaskOptions { - parent_hash, - parent_state_root, + parent_header, preserved_sparse_trie, transaction_count, config, @@ -536,6 +540,7 @@ impl DefaultStateRootStrategy { let (state_root_tx, state_root_rx) = mpsc::channel(); let (hashed_state_tx, hashed_state_rx) = mpsc::channel(); + let parent_state_root = parent_header.state_root(); self.spawn_sparse_trie_task( executor, @@ -546,8 +551,7 @@ impl DefaultStateRootStrategy { from_multi_proof, cancel_rx, SparseTrieTaskOptions { - parent_hash, - parent_state_root, + parent_header, preserved_sparse_trie, chunk_size: config.multiproof_chunk_size(), pending_sparse_trie_prune_blocks: if config.disable_sparse_trie_cache_pruning() { @@ -581,8 +585,7 @@ impl DefaultStateRootStrategy { options: SparseTrieTaskOptions, ) { let SparseTrieTaskOptions { - parent_hash, - parent_state_root, + parent_header, preserved_sparse_trie, chunk_size, pending_sparse_trie_prune_blocks, @@ -595,6 +598,12 @@ impl DefaultStateRootStrategy { executor.clone().spawn_blocking_named("sparse-trie", move || { reth_tasks::once!(increase_thread_priority); + let parent_hash = parent_header.hash(); + let parent_state_root = parent_header.state_root(); + let new_epoch = TrieNodeEpoch::new(parent_header.number().saturating_add(1)); + let prune_before = + sparse_trie_prune_before(pending_sparse_trie_prune_blocks.as_deref(), new_epoch); + let _enter = debug_span!( target: "engine::tree::payload_processor", parent: parent_span, @@ -651,6 +660,7 @@ impl DefaultStateRootStrategy { trie_metrics.clone(), sparse_state_trie, parent_state_root, + new_epoch, chunk_size, ); @@ -694,16 +704,14 @@ impl DefaultStateRootStrategy { let _enter = debug_span!(target: "engine::tree::payload_processor", "preserve").entered(); let mut trie_to_drop = None; - let deferred = if let Some(result) = task_result { + let deferred = if task_result.is_some() { let pending_trie = pending_trie.expect("pending trie is created for successful task result"); let start = Instant::now(); let (mut trie, deferred) = task.into_trie_for_reuse(); - if let Some(prune_blocks) = pending_sparse_trie_prune_blocks { + if let Some(prune_before) = prune_before { let prune_start = Instant::now(); - let retained_paths = - sparse_trie_retained_paths(prune_blocks, result.hashed_state.as_ref()); - trie.prune(retained_paths); + trie.prune(prune_before); trie_metrics .sparse_trie_prune_duration_histogram .record(prune_start.elapsed().as_secs_f64()); @@ -736,110 +744,31 @@ impl DefaultStateRootStrategy { } struct SparseTrieTaskOptions { - parent_hash: B256, - parent_state_root: B256, + parent_header: SealedHeader, preserved_sparse_trie: Option, chunk_size: usize, - /// `None` disables pruning. `Some(Vec::new())` prunes using only the current block's paths. + /// `None` disables pruning. `Some(Vec::new())` prunes nodes older than the current block. pending_sparse_trie_prune_blocks: Option>>, } struct StateRootTaskOptions<'a, N: NodePrimitives> { - parent_hash: B256, - parent_state_root: B256, + parent_header: SealedHeader, preserved_sparse_trie: Option, transaction_count: Option, config: &'a TreeConfig, pending_sparse_trie_prune_blocks: Option>>, } -fn sparse_trie_retained_paths( - prune_blocks: Vec>, - current_hashed_state: &HashedPostState, -) -> TriePrefixSets { - struct StorageAcc<'a> { - wiped: bool, - first: Option<&'a [(B256, U256)]>, - rest: Vec<&'a [(B256, U256)]>, - } - - let current_hashed_state = current_hashed_state.clone_into_sorted(); - let mut account_slices = Vec::with_capacity(prune_blocks.len() + 1); - let mut storage_acc = B256Map::>::default(); - - for state in prune_blocks - .iter() - .map(|block| block.trie_data.get().sorted.hashed_state.as_ref()) - .chain(core::iter::once(¤t_hashed_state)) - { - account_slices.push(state.accounts.as_slice()); - - for (address, storage) in &state.storages { - let entry = storage_acc.entry(*address).or_insert_with(|| StorageAcc { - wiped: false, - first: None, - rest: Vec::new(), - }); - if entry.wiped { - continue - } - if storage.wiped { - entry.wiped = true; - entry.first = None; - entry.rest.clear(); - } else if entry.first.is_some() { - entry.rest.push(storage.storage_slots.as_slice()); - } else { - entry.first = Some(storage.storage_slots.as_slice()); - } - } - } - - let mut storage_addresses = storage_acc.keys().copied().collect::>(); - storage_addresses.sort_unstable(); - - let (account_prefix_set, storage_prefix_sets) = join( - || { - PrefixSet::from( - account_slices - .into_iter() - .map(|slice| slice.iter().map(|(address, _)| address)) - .kmerge() - .dedup() - .copied() - .merge(storage_addresses) - .dedup(), - ) - }, - || { - storage_acc - .par_iter() - .map(|(address, entry)| { - let prefix_set = if entry.wiped { - PrefixSet::all_paths() - } else { - PrefixSet::from( - entry - .first - .iter() - .copied() - .chain(entry.rest.iter().copied()) - .map(|slice| slice.iter().map(|(slot, _)| slot)) - .kmerge() - .dedup() - .copied(), - ) - }; - (*address, prefix_set) - }) - .collect() - }, - ); - - TriePrefixSets { - account_prefix_set, - storage_prefix_sets, - destroyed_accounts: Default::default(), +fn sparse_trie_prune_before( + pending_sparse_trie_prune_blocks: Option<&[ExecutedBlock]>, + new_epoch: TrieNodeEpoch, +) -> Option { + // The parent chain is ordered newest to oldest. An empty chain means the block being + // calculated is the only in-memory block whose trie nodes need to be retained. + match pending_sparse_trie_prune_blocks { + None => None, + Some([]) => Some(new_epoch), + Some([.., oldest]) => Some(TrieNodeEpoch::new(oldest.recovered_block().number())), } } @@ -905,6 +834,7 @@ where executor, state_trie_overlays, env, + parent_header, provider_builder, overlay_factory, config, @@ -928,8 +858,7 @@ where state_trie_overlays, overlay_factory.clone(), StateRootTaskOptions { - parent_hash: env.parent_hash, - parent_state_root: env.parent_state_root, + parent_header: parent_header.clone(), preserved_sparse_trie, transaction_count: Some(env.transaction_count), config, @@ -988,6 +917,7 @@ where let pending_sparse_trie_prune_blocks = ctx.take_sparse_trie_prune_blocks(); let parent_state_root = ctx.parent_state_root(); + let parent_header = SealedHeader::new(ctx.parent_header().clone(), ctx.parent_hash()); let preserved_sparse_trie = ctx.state_trie_overlays.take_sparse_trie(); let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie .as_ref() @@ -1004,8 +934,7 @@ where ctx.state_trie_overlays, overlay_factory, StateRootTaskOptions { - parent_hash: ctx.parent_hash(), - parent_state_root, + parent_header, preserved_sparse_trie, // Tx count unknown at FCU time (block built incrementally): full proof workers. transaction_count: None, @@ -1391,128 +1320,23 @@ mod tests { HashingWriter, }; use reth_testing_utils::generators; - use reth_trie::{ - test_utils::state_root, HashedPostState, HashedStorage, LazyTrieData, Nibbles, - }; + use reth_trie::test_utils::state_root; use reth_trie_db::ChangesetCache; use revm::state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot, TransactionId}; - fn with_hashed_state( - block: ExecutedBlock, - hashed_state: HashedPostState, - ) -> ExecutedBlock { - let mut trie_data = block.trie_data(); - trie_data.sorted.hashed_state = Arc::new(hashed_state.into_sorted()); - ExecutedBlock::with_deferred_trie_data( - block.recovered_block, - block.execution_output, - LazyTrieData::ready(trie_data), - ) - } - - fn trie_hashed_state(account_path: u8, storage_path: u8) -> HashedPostState { - HashedPostState::default() - .with_accounts([(B256::with_last_byte(account_path), Some(Account::default()))]) - .with_storages([( - B256::with_last_byte(account_path), - HashedStorage::from_iter(false, [(B256::with_last_byte(storage_path), U256::ONE)]), - )]) - } - #[test] - fn sparse_trie_retained_paths_merges_prune_blocks_with_current_block() { - let blocks: Vec<_> = TestBlockBuilder::eth() - .get_executed_blocks(1..3) - .zip([trie_hashed_state(0x01, 0x02), trie_hashed_state(0x03, 0x04)]) - .map(|(block, hashed_state)| with_hashed_state(block, hashed_state)) - .collect(); - let current_hashed_state = trie_hashed_state(0x05, 0x06); - - let retained_paths = sparse_trie_retained_paths(blocks, ¤t_hashed_state); - - assert_eq!(retained_paths.account_prefix_set.len(), 3); - assert_eq!(retained_paths.storage_prefix_sets.len(), 3); - } - - #[test] - fn sparse_trie_retained_paths_uses_hashed_state() { - let blocks: Vec<_> = TestBlockBuilder::eth() - .get_executed_blocks(1..2) - .map(|block| with_hashed_state(block, trie_hashed_state(0x01, 0x02))) - .collect(); - let current_hashed_state = trie_hashed_state(0x03, 0x04); - - let retained_paths = sparse_trie_retained_paths(blocks, ¤t_hashed_state); - - assert_eq!(retained_paths.account_prefix_set.len(), 2); - assert_eq!(retained_paths.storage_prefix_sets.len(), 2); - } - - #[test] - fn sparse_trie_retained_paths_merges_sorted_paths_directly() { - let account = B256::with_last_byte(1); - let destroyed = B256::with_last_byte(2); - let storage_only = B256::with_last_byte(3); - let wiped_storage = B256::with_last_byte(4); - let account_slot = B256::with_last_byte(10); - let first_slot = B256::with_last_byte(11); - let second_slot = B256::with_last_byte(12); - let third_slot = B256::with_last_byte(13); - - let first = HashedPostState::default() - .with_accounts([(account, Some(Account::default())), (destroyed, None)]) - .with_storages([ - (account, HashedStorage::from_iter(false, [(account_slot, U256::from(1))])), - ( - storage_only, - HashedStorage::from_iter( - false, - [(first_slot, U256::from(1)), (third_slot, U256::from(3))], - ), - ), - (wiped_storage, HashedStorage::from_iter(false, [(first_slot, U256::from(1))])), - ]); - let second = HashedPostState::default().with_accounts([(account, None)]).with_storages([ - ( - storage_only, - HashedStorage::from_iter( - false, - [(second_slot, U256::from(2)), (third_slot, U256::from(30))], - ), - ), - (wiped_storage, HashedStorage::from_iter(true, [])), - ]); - let blocks = TestBlockBuilder::eth() - .get_executed_blocks(1..3) - .zip([first, second]) - .map(|(block, hashed_state)| with_hashed_state(block, hashed_state)) - .collect(); - - let retained_paths = sparse_trie_retained_paths(blocks, &HashedPostState::default()); - + fn sparse_trie_prune_before_uses_requested_range() { + let new_epoch = TrieNodeEpoch::new(10); + assert_eq!(sparse_trie_prune_before::(None, new_epoch), None); assert_eq!( - retained_paths.account_prefix_set.slice(), - &[ - Nibbles::unpack(account), - Nibbles::unpack(destroyed), - Nibbles::unpack(storage_only), - Nibbles::unpack(wiped_storage), - ] + sparse_trie_prune_before::(Some(&[]), new_epoch), + Some(new_epoch) ); - assert_eq!( - retained_paths.storage_prefix_sets[&storage_only].slice(), - &[ - Nibbles::unpack(first_slot), - Nibbles::unpack(second_slot), - Nibbles::unpack(third_slot), - ] - ); - assert_eq!( - retained_paths.storage_prefix_sets[&account].slice(), - &[Nibbles::unpack(account_slot)] - ); - assert!(retained_paths.storage_prefix_sets[&wiped_storage].all()); - assert!(retained_paths.destroyed_accounts.is_empty()); + + let mut blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(7..10).collect(); + blocks.reverse(); + + assert_eq!(sparse_trie_prune_before(Some(&blocks), new_epoch), Some(TrieNodeEpoch::new(7))); } #[test] @@ -1658,8 +1482,7 @@ mod tests { OverlayBuilder::::new(genesis_hash, ChangesetCache::new()), ), StateRootTaskOptions { - parent_hash: genesis_hash, - parent_state_root: env.parent_state_root, + parent_header: SealedHeader::new(Default::default(), genesis_hash), preserved_sparse_trie: None, transaction_count: Some(env.transaction_count), config: &TreeConfig::default(), diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 58214c6df58..ab9594bc85e 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -28,7 +28,7 @@ use reth_trie_parallel::{ use reth_trie_sparse::{ errors::{SparseStateTrieErrorKind, SparseTrieErrorKind, SparseTrieResult}, ArenaParallelSparseTrie, DeferredDrops, LeafUpdate, RevealableSparseTrie, SparseStateTrie, - SparseTrie, + SparseTrie, TrieNodeEpoch, }; use tracing::{debug, debug_span, error, instrument, trace_span}; @@ -50,6 +50,8 @@ pub(super) struct SparseTrieCacheTask, /// The parent block's state root. parent_state_root: B256, + /// The new epoch assigned to nodes modified by this task. + new_epoch: TrieNodeEpoch, /// Handle to the proof worker pools (storage and account). proof_worker_handle: ProofWorkerHandle, @@ -137,6 +139,7 @@ where metrics: SparseTrieTaskMetrics, trie: SparseStateTrie, parent_state_root: B256, + new_epoch: TrieNodeEpoch, chunk_size: usize, ) -> Self { let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); @@ -158,6 +161,7 @@ where final_hashed_state_tx: Some(final_hashed_state_tx), trie, parent_state_root, + new_epoch, chunk_size, max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING, account_updates: Default::default(), @@ -333,7 +337,7 @@ where debug!(target: "engine::root", "All proofs processed, ending calculation"); let start = Instant::now(); - let (state_root, trie_updates) = match self.trie.root_with_updates() { + let (state_root, trie_updates) = match self.trie.root_with_updates(self.new_epoch) { Ok(result) => result, Err(err) if matches!( @@ -429,7 +433,7 @@ where // If there's still no pending updates spend some time pre-computing the account // trie upper hashes if self.proof_result_rx.is_empty() { - self.trie.calculate_subtries(); + self.trie.calculate_subtries(self.new_epoch); } } else if !updates_queued { // If we don't have any pending updates, apply them to the trie, @@ -738,6 +742,7 @@ where let parent_span = debug_span!("compute_drained_storage_roots", n = tries_to_compute_roots.len()); + let new_epoch = self.new_epoch; tries_to_compute_roots.into_par_iter().for_each(|(address, SendStorageTriePtr(trie))| { let span = if tracing::enabled!(tracing::Level::TRACE) { debug_span!( @@ -760,7 +765,11 @@ where // - we do not insert/remove entries between pointer collection and use, so pointers // stay valid and map reallocation cannot occur; // - each pointer is consumed by at most one rayon task, so no aliasing mutable access. - unsafe { (*trie).root().expect("updates are drained, trie should be revealed by now") }; + unsafe { + (*trie) + .root(new_epoch) + .expect("updates are drained, trie should be revealed by now") + }; }); } @@ -792,7 +801,7 @@ where // If account has pending storage updates, it is still pending. return true; } else if let Some(account) = account.take() { - let storage_root = self.trie.storage_root(addr).expect("updates are drained, storage trie should be revealed by now"); + let storage_root = self.trie.storage_root(addr, self.new_epoch).expect("updates are drained, storage trie should be revealed by now"); let encoded = encode_account_leaf_value(account, storage_root, account_rlp_buf); self.account_updates.insert(*addr, LeafUpdate::Changed(encoded)); num_promoted += 1; @@ -821,7 +830,7 @@ where (account, storage_root) } else { - (trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now")) + (trie_account.map(Into::into), self.trie.storage_root(addr, self.new_epoch).expect("account had storage updates that were applied to its trie, storage root must be revealed by now")) }; let encoded = encode_account_leaf_value(account, storage_root, account_rlp_buf); @@ -978,7 +987,7 @@ pub(super) struct SparseTrieTaskMetrics { pub(super) sparse_trie_total_duration_histogram: Histogram, /// Time spent preparing the sparse trie for reuse after state root computation. pub(super) into_trie_for_reuse_duration_histogram: Histogram, - /// Time spent building the retention set and pruning the sparse trie. + /// Time spent pruning the sparse trie by node epoch. pub(super) sparse_trie_prune_duration_histogram: Histogram, /// Time spent waiting for preserved sparse trie cache to become available. pub(super) sparse_trie_cache_wait_duration_histogram: Histogram, @@ -1224,6 +1233,7 @@ mod tests { SparseTrieTaskMetrics::default(), trie, parent_state_root, + TrieNodeEpoch::UNMODIFIED, 1, ); @@ -1269,6 +1279,7 @@ mod tests { SparseTrieTaskMetrics::default(), trie, B256::from([0x55; 32]), + TrieNodeEpoch::UNMODIFIED, 1, ); @@ -1348,6 +1359,7 @@ mod tests { SparseTrieTaskMetrics::default(), trie, B256::from([0x55; 32]), + TrieNodeEpoch::UNMODIFIED, 1, ); @@ -1393,6 +1405,7 @@ mod tests { SparseTrieTaskMetrics::default(), trie, B256::from([0x55; 32]), + TrieNodeEpoch::UNMODIFIED, 1, ); diff --git a/crates/trie/sparse/src/arena/cursor.rs b/crates/trie/sparse/src/arena/cursor.rs index fbbd79e022c..d328e10bc1a 100644 --- a/crates/trie/sparse/src/arena/cursor.rs +++ b/crates/trie/sparse/src/arena/cursor.rs @@ -135,8 +135,7 @@ impl ArenaCursor { ArenaSparseNode::Leaf { state, .. } => matches!(state, ArenaSparseNodeState::Dirty), ArenaSparseNode::Subtrie(s) => { let root = &s.arena[s.root]; - matches!(root, ArenaSparseNode::EmptyRoot) || - matches!(root.state_ref(), Some(ArenaSparseNodeState::Dirty)) + matches!(root.state_ref(), Some(ArenaSparseNodeState::Dirty)) } _ => false, }); @@ -310,7 +309,7 @@ impl ArenaCursor { let head_idx = head.index; let head_branch = match &arena[head_idx] { - ArenaSparseNode::EmptyRoot => { + ArenaSparseNode::EmptyRoot { .. } => { return SeekResult::EmptyRoot; } ArenaSparseNode::Leaf { key, .. } => { diff --git a/crates/trie/sparse/src/arena/mod.rs b/crates/trie/sparse/src/arena/mod.rs index 03f613543f1..97eaf1e510b 100644 --- a/crates/trie/sparse/src/arena/mod.rs +++ b/crates/trie/sparse/src/arena/mod.rs @@ -8,7 +8,9 @@ use nodes::{ ArenaSparseNode, ArenaSparseNodeBranch, ArenaSparseNodeBranchChild, ArenaSparseNodeState, }; -use crate::{LeafLookup, LeafLookupError, LeafUpdate, SparseTrie, SparseTrieUpdates}; +use crate::{ + LeafLookup, LeafLookupError, LeafUpdate, SparseTrie, SparseTrieUpdates, TrieNodeEpoch, +}; use alloc::{borrow::Cow, boxed::Box, collections::VecDeque, vec::Vec}; use alloy_primitives::{keccak256, map::B256Map, B256}; use alloy_trie::TrieMask; @@ -36,26 +38,6 @@ const TRACE_TARGET: &str = "trie::arena"; /// depth or deeper belong to lower subtries. const UPPER_TRIE_MAX_DEPTH: usize = 2; -/// Finds the sub-range of `sorted_keys[start..]` whose entries start with `prefix`. -/// -/// Returns the half-open range `start_idx..end_idx` into `sorted_keys`. The returned -/// `end_idx` can be used as the `start` for the next call when iterating prefixes in -/// lexicographic order. -fn prefix_range( - sorted_keys: &[Nibbles], - start: usize, - prefix: &Nibbles, -) -> core::ops::Range { - // Advance past entries before `prefix`. - let begin = start + sorted_keys[start..].partition_point(|p| p < prefix); - // Find the end of entries that start with `prefix`. - let mut end = begin; - while end < sorted_keys.len() && sorted_keys[end].starts_with(prefix) { - end += 1; - } - begin..end -} - /// Compacts an arena by BFS-copying all reachable nodes into a fresh `SlotMap`, dropping /// unreachable (pruned) slots. Parents are stored before children for cache-friendly top-down /// traversal. @@ -158,7 +140,8 @@ impl ArenaSparseSubtrie { /// before use. fn new(record_updates: bool) -> Box { let mut arena = SlotMap::new(); - let root = arena.insert(ArenaSparseNode::EmptyRoot); + let root = + arena.insert(ArenaSparseNode::EmptyRoot { state: ArenaSparseNodeState::Revealed }); let buffers = ArenaTrieBuffers { updates: record_updates.then(SparseTrieUpdates::default), ..Default::default() @@ -191,15 +174,11 @@ impl ArenaSparseSubtrie { ); } - /// Prunes revealed subtrees that are not ancestors of any retained leaf, compacting the arena - /// in lexicographic order. - /// - /// `retained_leaves` must yield leaves in sorted order and be scoped to this subtrie's key - /// range. Builds a fresh arena by copying retained nodes from the root, blinding non-retained - /// children at the boundary unless their parent branch is retained. + /// Collapses nodes last modified before `prune_before` into hash stubs while copying retained + /// nodes into a compacted arena. /// /// Expects that all nodes have computed hashes (i.e. `prune` is called after hashing). - fn prune<'a>(&mut self, retained_leaves: impl IntoIterator) -> usize { + fn prune(&mut self, prune_before: TrieNodeEpoch) -> usize { // Only branches can have pruneable children. if !matches!(&self.arena[self.root], ArenaSparseNode::Branch(_)) { return 0; @@ -207,29 +186,22 @@ impl ArenaSparseSubtrie { debug_assert_eq!(self.num_dirty_leaves, 0, "prune must run after hashing"); - let retained_leaves = retained_leaves.into_iter(); - let (retained_lower_bound, retained_upper_bound) = retained_leaves.size_hint(); + if prune_before == TrieNodeEpoch::UNMODIFIED { + return 0; + } + let old_count = self.arena.len(); - // In a tree where every branch has ≥2 children, #branches ≤ #leaves − 1, so - // total nodes ≤ 2N − 1. This is a reasonable upper-bound capacity hint that - // avoids most reallocations without over-allocating when pruning is heavy. - let mut new_arena = - SlotMap::with_capacity(retained_upper_bound.unwrap_or(retained_lower_bound) * 2); - let mut retained_leaves = retained_leaves.peekable(); + // Do not reserve the old arena's size: discarded nodes should release their capacity. + let mut new_arena = SlotMap::new(); let mut new_num_leaves = 0u64; - // Root is always retained. + // The subtrie root is retained by the owning upper trie. let root_node = self.arena.remove(self.root).expect("root exists"); let new_root = new_arena.insert(root_node); let mut stack = Vec::new(); - let root_is_retained = retained_leaves.peek().is_some(); - if let Some(frame) = prepare_retained_node( - &new_arena, - new_root, - self.path, - root_is_retained, - &mut new_num_leaves, - ) { + if let Some(frame) = + prepare_retained_node(&new_arena, new_root, self.path, &mut new_num_leaves) + { stack.push(frame); } @@ -240,43 +212,22 @@ impl ArenaSparseSubtrie { continue; }; - // Child's path in the trie (edges to reach it, excluding its own short_key). let parent_new_idx = frame.new_idx; let mut child_path = frame.branch_logical_path; child_path.push(nibble); - while retained_leaves.peek().is_some_and(|retained| *retained < &child_path) { - retained_leaves.next(); - } + let child_epoch = self.arena[old_child_idx] + .state_ref() + .and_then(ArenaSparseNodeState::cached_epoch) + .expect("prune must run after hashing"); - let child_is_retained = - retained_leaves.peek().is_some_and(|retained| retained.starts_with(&child_path)); - if child_is_retained || frame.branch_is_retained { - // Retained or protected by a retained parent branch. - let child_node = self.arena.remove(old_child_idx).expect("child exists"); - let new_child_idx = new_arena.insert(child_node); - if let Some(frame) = prepare_retained_node( - &new_arena, - new_child_idx, - child_path, - child_is_retained, - &mut new_num_leaves, - ) { - stack.push(frame); - } - let ArenaSparseNode::Branch(b) = &mut new_arena[parent_new_idx] else { - unreachable!() - }; - b.children[child_pos] = ArenaSparseNodeBranchChild::Revealed(new_child_idx); - } else { - // Not retained — blind the child slot in the new arena. + if child_epoch.should_prune(prune_before) { let node = &self.arena[old_child_idx]; let rlp_node = node .state_ref() - .expect("child must have state") - .cached_rlp_node() + .and_then(ArenaSparseNodeState::cached_rlp_node) .cloned() - .expect("pruned child must have cached RLP (prune runs after hashing)"); + .expect("prune must run after hashing"); trace!( target: TRACE_TARGET, path = ?child_path, @@ -288,6 +239,21 @@ impl ArenaSparseSubtrie { unreachable!() }; b.children[child_pos] = ArenaSparseNodeBranchChild::Blinded(rlp_node); + } else { + let child_node = self.arena.remove(old_child_idx).expect("child exists"); + let new_child_idx = new_arena.insert(child_node); + if let Some(frame) = prepare_retained_node( + &new_arena, + new_child_idx, + child_path, + &mut new_num_leaves, + ) { + stack.push(frame); + } + let ArenaSparseNode::Branch(b) = &mut new_arena[parent_new_idx] else { + unreachable!() + }; + b.children[child_pos] = ArenaSparseNodeBranchChild::Revealed(new_child_idx); } } @@ -306,7 +272,6 @@ impl ArenaSparseSubtrie { branch_logical_path: Nibbles, state_mask: TrieMask, remaining_child_mask: TrieMask, - branch_is_retained: bool, } impl CopyFrame { @@ -326,13 +291,12 @@ impl ArenaSparseSubtrie { } } - /// Prepares a retained node for lexicographic copying, returning a stack frame when the - /// node has children to walk. + /// Prepares a retained node for copying, returning a stack frame when the node has children + /// to walk. fn prepare_retained_node( new_arena: &NodeArena, new_idx: Index, node_path: Nibbles, - branch_is_retained: bool, new_num_leaves: &mut u64, ) -> Option { let ArenaSparseNode::Branch(b) = &new_arena[new_idx] else { @@ -342,7 +306,6 @@ impl ArenaSparseSubtrie { return None; }; - // Logical path of this branch (path TO node + its extension/short_key). let mut branch_logical_path = node_path; branch_logical_path.extend(&b.short_key); @@ -351,7 +314,6 @@ impl ArenaSparseSubtrie { branch_logical_path, state_mask: b.state_mask, remaining_child_mask: b.state_mask, - branch_is_retained, }) } } @@ -379,7 +341,7 @@ impl ArenaSparseSubtrie { trace!(target: TRACE_TARGET, "Subtrie update_leaves"); debug_assert!( - !matches!(self.arena[self.root], ArenaSparseNode::EmptyRoot), + !matches!(self.arena[self.root], ArenaSparseNode::EmptyRoot { .. }), "subtrie root must not be EmptyRoot at start of update_leaves" ); @@ -453,7 +415,7 @@ impl ArenaSparseSubtrie { trace!(target: TRACE_TARGET, path = ?self.path, num_nodes = nodes.len(), "Subtrie reveal_nodes"); debug_assert!( - !matches!(self.arena[self.root], ArenaSparseNode::EmptyRoot), + !matches!(self.arena[self.root], ArenaSparseNode::EmptyRoot { .. }), "subtrie root must not be EmptyRoot in reveal_nodes" ); @@ -486,12 +448,13 @@ impl ArenaSparseSubtrie { /// After this call every node reachable from `self.root` will be in `Cached` state. /// /// Trie updates are written directly to `self.buffers.updates` (if `Some`). - fn update_cached_rlp(&mut self) { + fn update_cached_rlp(&mut self, new_epoch: TrieNodeEpoch) { ArenaParallelSparseTrie::update_cached_rlp( &mut self.arena, self.root, self.path, &mut self.buffers, + new_epoch, ); self.num_dirty_leaves = 0; #[cfg(debug_assertions)] @@ -630,9 +593,9 @@ impl Default for ArenaParallelismThresholds { /// /// ## Pruning /// -/// [`SparseTrie::prune`] removes revealed nodes that are not ancestors of any retained leaf. -/// Pruned nodes are replaced with `ArenaSparseNodeBranchChild::Blinded` entries using their -/// cached RLP. Subtries are pruned in parallel when their leaf count exceeds +/// [`SparseTrie::prune`] replaces nodes older than its epoch cutoff with +/// `ArenaSparseNodeBranchChild::Blinded` entries using their cached RLP, then compacts the +/// arenas. Subtries are pruned in parallel when their leaf count exceeds /// [`ArenaParallelismThresholds::min_leaves_for_prune`]. #[derive(Debug, Clone)] pub struct ArenaParallelSparseTrie { @@ -689,7 +652,7 @@ impl ArenaParallelSparseTrie { path: Nibbles, ) -> Option { match &arena[idx] { - ArenaSparseNode::EmptyRoot => Some(ProofTrieNodeRecord { + ArenaSparseNode::EmptyRoot { .. } => Some(ProofTrieNodeRecord { path, node: TrieNodeRecord(TrieNode::EmptyRoot), masks: None, @@ -903,7 +866,7 @@ impl ArenaParallelSparseTrie { return; }; - if !matches!(subtrie.arena[subtrie.root], ArenaSparseNode::EmptyRoot) { + if !matches!(subtrie.arena[subtrie.root], ArenaSparseNode::EmptyRoot { .. }) { return; } @@ -983,7 +946,8 @@ impl ArenaParallelSparseTrie { if count == 0 { if branch_idx == self.root { - self.upper_arena[branch_idx] = ArenaSparseNode::EmptyRoot; + self.upper_arena[branch_idx] = + ArenaSparseNode::EmptyRoot { state: ArenaSparseNodeState::Dirty }; return; } // Remove the empty branch from its parent. @@ -1025,7 +989,7 @@ impl ArenaParallelSparseTrie { // Check if the remaining child is an empty subtrie that should also be removed. let is_empty_subtrie = matches!( &self.upper_arena[child_idx], - ArenaSparseNode::Subtrie(s) if matches!(s.arena[s.root], ArenaSparseNode::EmptyRoot) + ArenaSparseNode::Subtrie(s) if matches!(s.arena[s.root], ArenaSparseNode::EmptyRoot { .. }) ); if is_empty_subtrie { @@ -1151,6 +1115,7 @@ impl ArenaParallelSparseTrie { root: Index, base_path: Nibbles, buffers: &mut ArenaTrieBuffers, + new_epoch: TrieNodeEpoch, ) -> RlpNode { let cursor = &mut buffers.cursor; let rlp_buf = &mut buffers.rlp_buf; @@ -1160,12 +1125,22 @@ impl ArenaParallelSparseTrie { rlp_node_buf.clear(); // Step 1: Handle trivial roots that don't need the stack-based walk. - // EmptyRoot has no state to update. Leaves are encoded in place. Already-cached - // branches need no work. Only dirty branches enter the main loop below. + // Empty roots and leaves are encoded in place. Already-cached branches need no work. + // Only dirty branches enter the main loop below. match &arena[root] { - ArenaSparseNode::EmptyRoot => return RlpNode::word_rlp(&EMPTY_ROOT_HASH), + ArenaSparseNode::EmptyRoot { state } => { + let node_epoch = match state { + ArenaSparseNodeState::Cached { epoch, .. } => *epoch, + ArenaSparseNodeState::Revealed => TrieNodeEpoch::UNMODIFIED, + ArenaSparseNodeState::Dirty => new_epoch, + }; + let rlp_node = RlpNode::word_rlp(&EMPTY_ROOT_HASH); + *arena[root].state_mut() = + ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone(), epoch: node_epoch }; + return rlp_node + } ArenaSparseNode::Leaf { .. } => { - Self::encode_leaf(arena, root, rlp_buf, rlp_node_buf); + Self::encode_leaf(arena, root, rlp_buf, rlp_node_buf, new_epoch); return rlp_node_buf.pop().expect("encode_leaf must push an RlpNode"); } ArenaSparseNode::Branch(b) => { @@ -1216,6 +1191,7 @@ impl ArenaParallelSparseTrie { ); rlp_node_buf.clear(); + let mut node_epoch = TrieNodeEpoch::UNMODIFIED; let state_mask = arena[head_idx].branch_ref().state_mask; for (child_idx, _nibble) in BranchChildIter::new(state_mask) { match &arena[head_idx].branch_ref().children[child_idx] { @@ -1226,7 +1202,13 @@ impl ArenaParallelSparseTrie { let child_idx = *child_idx; match &arena[child_idx] { ArenaSparseNode::Leaf { .. } => { - Self::encode_leaf(arena, child_idx, rlp_buf, rlp_node_buf); + Self::encode_leaf( + arena, + child_idx, + rlp_buf, + rlp_node_buf, + new_epoch, + ); } ArenaSparseNode::Branch(child_b) => { let ArenaSparseNodeState::Cached { rlp_node, .. } = &child_b.state @@ -1252,10 +1234,16 @@ impl ArenaParallelSparseTrie { _ => panic!("subtrie root must be a cached Branch or Leaf"), } } - ArenaSparseNode::TakenSubtrie | ArenaSparseNode::EmptyRoot => { + ArenaSparseNode::TakenSubtrie | ArenaSparseNode::EmptyRoot { .. } => { unreachable!("Unexpected child {:?}", arena[child_idx]); } } + let Some(ArenaSparseNodeState::Cached { epoch: child_epoch, .. }) = + arena[child_idx].state_ref() + else { + panic!("revealed child must be cached after encoding"); + }; + node_epoch = node_epoch.max(*child_epoch); } } } @@ -1267,6 +1255,9 @@ impl ArenaParallelSparseTrie { let prev_branch_masks = b.branch_masks; let new_branch_masks = Self::get_branch_masks(arena, b); let was_dirty = matches!(b.state, ArenaSparseNodeState::Dirty); + if was_dirty { + node_epoch = node_epoch.max(new_epoch); + } rlp_buf.clear(); let rlp_node = BranchNodeRef::new(rlp_node_buf, state_mask).rlp(rlp_buf); @@ -1288,7 +1279,8 @@ impl ArenaParallelSparseTrie { ); let branch = arena[head_idx].branch_mut(); - branch.state = ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone() }; + branch.state = + ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone(), epoch: node_epoch }; branch.branch_masks = new_branch_masks; // Record trie updates for dirty branches only. @@ -1326,7 +1318,7 @@ impl ArenaParallelSparseTrie { ) -> Option<&'a Vec> { loop { match &arena[current] { - ArenaSparseNode::EmptyRoot | ArenaSparseNode::TakenSubtrie => return None, + ArenaSparseNode::EmptyRoot { .. } | ArenaSparseNode::TakenSubtrie => return None, ArenaSparseNode::Leaf { key, value, .. } => { let remaining = full_path.slice(path_offset..); return (remaining == *key).then_some(value); @@ -1374,7 +1366,7 @@ impl ArenaParallelSparseTrie { ) -> Result { loop { match &arena[current] { - ArenaSparseNode::EmptyRoot | ArenaSparseNode::TakenSubtrie => { + ArenaSparseNode::EmptyRoot { .. } | ArenaSparseNode::TakenSubtrie => { return Ok(LeafLookup::NonExistent); } ArenaSparseNode::Leaf { key, value, .. } => { @@ -1446,21 +1438,27 @@ impl ArenaParallelSparseTrie { idx: Index, rlp_buf: &mut Vec, rlp_node_buf: &mut Vec, + new_epoch: TrieNodeEpoch, ) { let (key, value, state) = match &arena[idx] { ArenaSparseNode::Leaf { key, value, state } => (key, value, state), _ => unreachable!("encode_leaf called on non-Leaf node"), }; - if let ArenaSparseNodeState::Cached { rlp_node, .. } = state { - rlp_node_buf.push(rlp_node.clone()); - return; - } + let epoch = match state { + ArenaSparseNodeState::Cached { rlp_node, .. } => { + rlp_node_buf.push(rlp_node.clone()); + return; + } + ArenaSparseNodeState::Revealed => TrieNodeEpoch::UNMODIFIED, + ArenaSparseNodeState::Dirty => new_epoch, + }; rlp_buf.clear(); let rlp_node = LeafNodeRef { key, value }.rlp(rlp_buf); - *arena[idx].state_mut() = ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone() }; + *arena[idx].state_mut() = + ArenaSparseNodeState::Cached { rlp_node: rlp_node.clone(), epoch }; rlp_node_buf.push(rlp_node); } @@ -1751,7 +1749,8 @@ impl ArenaParallelSparseTrie { // The leaf is the root — replace with EmptyRoot and reset the cursor // so subsequent iterations can call seek normally. arena.remove(head_idx); - *root = arena.insert(ArenaSparseNode::EmptyRoot); + *root = arena + .insert(ArenaSparseNode::EmptyRoot { state: ArenaSparseNodeState::Dirty }); cursor.reset(arena, *root, head_path); return ( RemoveLeafResult::Removed, @@ -2076,8 +2075,7 @@ impl ArenaParallelSparseTrie { } /// Removes a pruned node from the arena and blinds the parent's child slot with the node's - /// cached RLP. If the node has no cached RLP (e.g. it was never hashed), the parent slot - /// is left dangling — this is safe because the parent will also be removed during pruning. + /// cached RLP. fn remove_pruned_node( arena: &mut NodeArena, cursor: &ArenaCursor, @@ -2086,7 +2084,11 @@ impl ArenaParallelSparseTrie { ) -> ArenaSparseNode { let path = cursor.head().expect("cursor is non-empty").path; let node = arena.remove(idx).expect("node must exist to be pruned"); - let rlp_node = node.state_ref().and_then(|s| s.cached_rlp_node()).cloned(); + let rlp_node = node + .state_ref() + .and_then(ArenaSparseNodeState::cached_rlp_node) + .cloned() + .expect("prune must run after hashing"); trace!( target: TRACE_TARGET, ?path, @@ -2095,14 +2097,12 @@ impl ArenaParallelSparseTrie { "pruning node", ); - if let Some(rlp_node) = rlp_node { - let parent_idx = cursor.parent().expect("pruned child has parent").index; - let child_nibble = nibble.expect("non-root child"); - let parent_branch = arena[parent_idx].branch_mut(); - let child_idx = BranchChildIdx::new(parent_branch.state_mask, child_nibble) - .expect("child nibble not found in parent state_mask"); - parent_branch.children[child_idx] = ArenaSparseNodeBranchChild::Blinded(rlp_node); - } + let parent_idx = cursor.parent().expect("pruned child has parent").index; + let child_nibble = nibble.expect("non-root child"); + let parent_branch = arena[parent_idx].branch_mut(); + let child_idx = BranchChildIdx::new(parent_branch.state_mask, child_nibble) + .expect("child nibble not found in parent state_mask"); + parent_branch.children[child_idx] = ArenaSparseNodeBranchChild::Blinded(rlp_node); node } @@ -2161,7 +2161,8 @@ impl ArenaParallelSparseTrie { let mut arena_node = ArenaSparseNode::from_proof_node(proof_node); let state = arena_node.state_mut(); - *state = ArenaSparseNodeState::Cached { rlp_node: cached_rlp }; + *state = + ArenaSparseNodeState::Cached { rlp_node: cached_rlp, epoch: TrieNodeEpoch::UNMODIFIED }; let child_idx = arena.insert(arena_node); arena[head_idx].branch_mut().children[dense_child_idx] = @@ -2223,7 +2224,8 @@ impl Drop for ArenaParallelSparseTrie { impl Default for ArenaParallelSparseTrie { fn default() -> Self { let mut upper_arena = SlotMap::new(); - let root = upper_arena.insert(ArenaSparseNode::EmptyRoot); + let root = upper_arena + .insert(ArenaSparseNode::EmptyRoot { state: ArenaSparseNodeState::Revealed }); Self { upper_arena, root, @@ -2237,13 +2239,13 @@ impl Default for ArenaParallelSparseTrie { impl ArenaParallelSparseTrie { /// Hashes a subtrie at `head_idx` and collects its update actions. - fn update_upper_subtrie(&mut self, head_idx: Index) { + fn update_upper_subtrie(&mut self, head_idx: Index, new_epoch: TrieNodeEpoch) { let ArenaSparseNode::Subtrie(subtrie) = &mut self.upper_arena[head_idx] else { unreachable!() }; if !subtrie.arena[subtrie.root].is_cached() { - subtrie.update_cached_rlp(); + subtrie.update_cached_rlp(new_epoch); } Self::merge_subtrie_updates(&mut self.buffers.updates, &mut subtrie.buffers.updates); @@ -2268,7 +2270,7 @@ impl SparseTrie for ArenaParallelSparseTrie { }); debug_assert!( - matches!(self.upper_arena[self.root], ArenaSparseNode::EmptyRoot), + matches!(self.upper_arena[self.root], ArenaSparseNode::EmptyRoot { .. }), "set_root called on a trie that already has revealed nodes" ); @@ -2277,6 +2279,8 @@ impl SparseTrie for ArenaParallelSparseTrie { match root { TrieNodeV2::EmptyRoot => { trace!(target: TRACE_TARGET, "Setting empty root"); + self.upper_arena[self.root] = + ArenaSparseNode::EmptyRoot { state: ArenaSparseNodeState::Revealed }; } TrieNodeV2::Leaf(leaf) => { trace!(target: TRACE_TARGET, key = ?leaf.key, "Setting leaf root"); @@ -2329,7 +2333,7 @@ impl SparseTrie for ArenaParallelSparseTrie { nodes: nodes.iter().map(ProofTrieNodeRecord::from_proof_trie_node_v2).collect(), }); - if matches!(self.upper_arena[self.root], ArenaSparseNode::EmptyRoot) { + if matches!(self.upper_arena[self.root], ArenaSparseNode::EmptyRoot { .. }) { trace!(target: TRACE_TARGET, "Skipping reveal_nodes on empty root"); return Ok(()); } @@ -2463,17 +2467,18 @@ impl SparseTrie for ArenaParallelSparseTrie { } #[instrument(level = "trace", target = TRACE_TARGET, skip_all, ret)] - fn root(&mut self) -> B256 { + fn root(&mut self, new_epoch: TrieNodeEpoch) -> B256 { #[cfg(feature = "trie-debug")] self.debug_recorder.record(RecordedOp::Root); - self.update_subtrie_hashes(); + self.update_subtrie_hashes(new_epoch); let rlp_node = Self::update_cached_rlp( &mut self.upper_arena, self.root, Nibbles::default(), &mut self.buffers, + new_epoch, ); rlp_node.as_hash().expect("root RlpNode must be a hash") @@ -2483,8 +2488,16 @@ impl SparseTrie for ArenaParallelSparseTrie { self.upper_arena[self.root].is_cached() } + fn root_epoch(&self) -> Option { + match self.upper_arena[self.root].state_ref()? { + ArenaSparseNodeState::Revealed => Some(TrieNodeEpoch::UNMODIFIED), + ArenaSparseNodeState::Cached { epoch, .. } => Some(*epoch), + ArenaSparseNodeState::Dirty => None, + } + } + #[instrument(level = "trace", target = TRACE_TARGET, skip_all)] - fn update_subtrie_hashes(&mut self) { + fn update_subtrie_hashes(&mut self, new_epoch: TrieNodeEpoch) { #[cfg(feature = "trie-debug")] self.debug_recorder.record(RecordedOp::UpdateSubtrieHashes); @@ -2517,7 +2530,7 @@ impl SparseTrie for ArenaParallelSparseTrie { if taken.len() == 1 || total_dirty_leaves < self.parallelism_thresholds.min_dirty_leaves { for (_, subtrie) in &mut taken { - subtrie.update_cached_rlp(); + subtrie.update_cached_rlp(new_epoch); } } else { use rayon::iter::{IntoParallelIterator, ParallelIterator}; @@ -2527,7 +2540,7 @@ impl SparseTrie for ArenaParallelSparseTrie { .into_par_iter() .map(|(idx, mut subtrie)| { let _guard = parent_span.enter(); - subtrie.update_cached_rlp(); + subtrie.update_cached_rlp(new_epoch); (idx, subtrie) }) .collect(); @@ -2573,7 +2586,7 @@ impl SparseTrie for ArenaParallelSparseTrie { self.upper_arena[head_idx] = ArenaSparseNode::Subtrie(subtrie); } - self.update_upper_subtrie(head_idx); + self.update_upper_subtrie(head_idx, new_epoch); } } @@ -2613,6 +2626,7 @@ impl SparseTrie for ArenaParallelSparseTrie { fn wipe(&mut self) { trace!(target: TRACE_TARGET, "Wiping arena trie"); self.clear(); + *self.upper_arena[self.root].state_mut() = ArenaSparseNodeState::Dirty; self.buffers.updates = self.buffers.updates.is_some().then(SparseTrieUpdates::wiped); } @@ -2622,48 +2636,35 @@ impl SparseTrie for ArenaParallelSparseTrie { self.debug_recorder.reset(); self.upper_arena = SlotMap::new(); - self.root = self.upper_arena.insert(ArenaSparseNode::EmptyRoot); + self.root = self + .upper_arena + .insert(ArenaSparseNode::EmptyRoot { state: ArenaSparseNodeState::Revealed }); self.buffers.clear(); } - fn size_hint(&self) -> usize { - self.upper_arena - .iter() - .map(|(_, node)| match node { - ArenaSparseNode::Subtrie(s) => s.num_leaves as usize, - ArenaSparseNode::Leaf { .. } => 1, - _ => 0, - }) - .sum() - } - #[instrument( level = "trace", target = TRACE_TARGET, skip_all, - fields(num_retained_leaves = retained_leaves.len()), + fields(prune_before = prune_before.get()), )] - fn prune(&mut self, retained_leaves: &[Nibbles]) -> usize { + fn prune(&mut self, prune_before: TrieNodeEpoch) -> usize { + assert!(self.root_epoch().is_some(), "prune cannot run on a dirty trie"); + // Only descend if the root is a branch; otherwise there are no subtries. if !matches!(&self.upper_arena[self.root], ArenaSparseNode::Branch(_)) { return 0; } - debug_assert!( - retained_leaves.windows(2).all(|w| w[0] <= w[1]), - "retained_leaves must be sorted" - ); - let threshold = self.parallelism_thresholds.min_leaves_for_prune; let mut cursor = mem::take(&mut self.buffers.cursor); cursor.reset(&self.upper_arena, self.root, Nibbles::default()); - // Subtries taken for parallel pruning: (arena_index, subtrie, retained_range). - let mut taken: Vec<(Index, Box, core::ops::Range)> = Vec::new(); + // Subtries taken for parallel pruning. + let mut taken: Vec<(Index, Box)> = Vec::new(); let mut pruned = 0; - let mut retained_idx = 0; loop { let result = cursor.next(&mut self.upper_arena, |_, child| { @@ -2675,20 +2676,13 @@ impl SparseTrie for ArenaParallelSparseTrie { ) }); - match result { - NextResult::Done => break, - NextResult::NonBranch | NextResult::Branch => {} + if matches!(result, NextResult::Done) { + break } let head = cursor.head().expect("cursor is non-empty"); let head_idx = head.index; let head_path = head.path; - let protected_by_retained_parent = if cursor.depth() == 0 { - false - } else { - let parent_path = cursor.parent().expect("cursor must have a parent").path; - !prefix_range(retained_leaves, 0, &parent_path).is_empty() - }; match &self.upper_arena[head_idx] { ArenaSparseNode::Branch(_) | ArenaSparseNode::Leaf { .. } => { @@ -2697,12 +2691,11 @@ impl SparseTrie for ArenaParallelSparseTrie { continue; } - let range = prefix_range(retained_leaves, 0, &head_path); - if !range.is_empty() { - continue; - } - - if protected_by_retained_parent { + let node_epoch = self.upper_arena[head_idx] + .state_ref() + .and_then(ArenaSparseNodeState::cached_epoch) + .expect("prune must run after hashing"); + if !node_epoch.should_prune(prune_before) { continue; } @@ -2715,19 +2708,11 @@ impl SparseTrie for ArenaParallelSparseTrie { pruned += 1; } ArenaSparseNode::Subtrie(_) => { - let subtrie_range = prefix_range(retained_leaves, retained_idx, &head_path); - retained_idx = subtrie_range.end; - - if subtrie_range.is_empty() { - if protected_by_retained_parent { - let ArenaSparseNode::Subtrie(subtrie) = &mut self.upper_arena[head_idx] - else { - unreachable!() - }; - pruned += subtrie.prune(&[]); - continue; - } - + let root_epoch = self.upper_arena[head_idx] + .state_ref() + .and_then(ArenaSparseNodeState::cached_epoch) + .expect("prune must run after hashing"); + if root_epoch.should_prune(prune_before) { let removed = Self::remove_pruned_node( &mut self.upper_arena, &cursor, @@ -2735,7 +2720,7 @@ impl SparseTrie for ArenaParallelSparseTrie { head_path.last(), ); let ArenaSparseNode::Subtrie(s) = &removed else { unreachable!() }; - pruned += s.num_leaves as usize; + pruned += s.arena.len(); self.recycle_subtrie(removed); continue; } @@ -2750,13 +2735,13 @@ impl SparseTrie for ArenaParallelSparseTrie { ) else { unreachable!() }; - taken.push((head_idx, subtrie, subtrie_range)); + taken.push((head_idx, subtrie)); } else { let ArenaSparseNode::Subtrie(subtrie) = &mut self.upper_arena[head_idx] else { unreachable!() }; - pruned += subtrie.prune(retained_leaves[subtrie_range].iter()); + pruned += subtrie.prune(prune_before); } } _ => unreachable!("NonBranch in prune walk must be Subtrie, Leaf, or Branch"), @@ -2768,15 +2753,15 @@ impl SparseTrie for ArenaParallelSparseTrie { if !taken.is_empty() { // Prune taken subtries, in parallel if more than one. if taken.len() == 1 { - let (_, ref mut subtrie, ref range) = taken[0]; - pruned += subtrie.prune(retained_leaves[range.clone()].iter()); + let (_, ref mut subtrie) = taken[0]; + pruned += subtrie.prune(prune_before); } else { use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; let parent_span = tracing::Span::current(); pruned += taken .par_iter_mut() - .map(|(_, subtrie, range)| { + .map(|(_, subtrie)| { let _guard = parent_span.enter(); let _span = tracing::trace_span!( target: TRACE_TARGET, @@ -2785,13 +2770,13 @@ impl SparseTrie for ArenaParallelSparseTrie { ) .entered(); - subtrie.prune(retained_leaves[range.clone()].iter()) + subtrie.prune(prune_before) }) .sum::(); } // Restore taken subtries into the upper arena. - for (child_idx, subtrie, _) in taken { + for (child_idx, subtrie) in taken { self.upper_arena[child_idx] = ArenaSparseNode::Subtrie(subtrie); } } @@ -3093,7 +3078,7 @@ impl SparseTrie for ArenaParallelSparseTrie { let head_idx = cursor.head().expect("cursor is non-empty").index; !matches!( &self.upper_arena[head_idx], - ArenaSparseNode::Subtrie(s) if matches!(s.arena[s.root], ArenaSparseNode::EmptyRoot) + ArenaSparseNode::Subtrie(s) if matches!(s.arena[s.root], ArenaSparseNode::EmptyRoot { .. }) ) }, "taken subtrie became EmptyRoot — should have been forced inline" @@ -3138,14 +3123,20 @@ impl SparseTrie for ArenaParallelSparseTrie { #[cfg(test)] mod tests { use super::TRACE_TARGET; - use crate::{ArenaParallelSparseTrie, ArenaParallelismThresholds, LeafUpdate, SparseTrie}; + use crate::{ + ArenaParallelSparseTrie, ArenaParallelismThresholds, LeafUpdate, SparseTrie, TrieNodeEpoch, + }; use alloy_primitives::{map::B256Map, B256, U256}; use rand::{seq::SliceRandom, Rng, SeedableRng}; use reth_trie::test_utils::TrieTestHarness; - use reth_trie_common::{Nibbles, ProofV2Target}; + use reth_trie_common::ProofV2Target; use std::collections::BTreeMap; use tracing::{info, trace}; + const fn epoch(value: u64) -> TrieNodeEpoch { + TrieNodeEpoch::new(value) + } + /// Test harness for proptest-based arena sparse trie testing. /// /// Wraps [`TrieTestHarness`] and adds `ArenaParallelSparseTrie`-specific helpers for @@ -3223,7 +3214,7 @@ mod tests { } // Compute root and take updates from the APST. - let actual_root = apst.root(); + let actual_root = apst.root(epoch(0)); let mut actual_updates = apst.take_updates(); // Minimize sparse updates inline (can't use TrieTestHarness::minimize_sparse_updates @@ -3326,17 +3317,9 @@ mod tests { // Update the harness base dataset to reflect the first changeset. harness.apply_changeset(changeset1); - // Pick N random keys from the current storage as retained leaves for pruning. - let mut all_storage_keys: Vec = harness.storage().keys() - .map(|k| Nibbles::unpack(*k)) - .collect(); - all_storage_keys.shuffle(&mut rng); - let num_retain = if all_storage_keys.is_empty() { 0 } else { - rng.random_range(0..=all_storage_keys.len()) - }; - let mut retained: Vec = all_storage_keys[..num_retain].to_vec(); - retained.sort_unstable(); - apst.prune(&retained); + // All nodes were cached at epoch 0, so this maximally prunes the trie before the + // second update round. + apst.prune(epoch(1)); let changeset2 = build_changeset(harness.storage(), changeset2_new_keys, overlap_pct, delete_pct, &mut rng); for (i, (k, v)) in changeset2.iter().enumerate() { diff --git a/crates/trie/sparse/src/arena/nodes.rs b/crates/trie/sparse/src/arena/nodes.rs index 550d2b7444d..a00d271ebf8 100644 --- a/crates/trie/sparse/src/arena/nodes.rs +++ b/crates/trie/sparse/src/arena/nodes.rs @@ -9,6 +9,8 @@ use reth_trie_common::{BranchNodeMasks, Nibbles, ProofTrieNodeV2, RlpNode, TrieN use smallvec::SmallVec; use strum::AsRefStr; +use crate::TrieNodeEpoch; + /// Tracks whether a node's RLP encoding is cached or needs recomputation. #[derive(Debug, Clone, PartialEq, Eq)] pub(super) enum ArenaSparseNodeState { @@ -18,6 +20,8 @@ pub(super) enum ArenaSparseNodeState { Cached { /// The cached RLP-encoded representation of the node. rlp_node: RlpNode, + /// The newest tracked modification epoch for this node or its descendants. + epoch: TrieNodeEpoch, }, /// The node has been modified and its RLP encoding needs recomputation. Dirty, @@ -36,6 +40,14 @@ impl ArenaSparseNodeState { _ => None, } } + + /// Returns the cached epoch, if this node is cached. + pub(super) const fn cached_epoch(&self) -> Option { + match self { + Self::Cached { epoch, .. } => Some(*epoch), + _ => None, + } + } } /// Represents a reference from a branch node to one of its children. @@ -155,7 +167,10 @@ impl ArenaSparseNodeBranch { #[derive(Debug, Clone, AsRefStr)] pub(super) enum ArenaSparseNode { /// Indicates a trie with no nodes. - EmptyRoot, + EmptyRoot { + /// Cached or dirty state of this node. Its cached RLP is always the empty root hash. + state: ArenaSparseNodeState, + }, /// A branch node with up to 16 children. Branch(ArenaSparseNodeBranch), /// A leaf node containing a value. @@ -174,26 +189,27 @@ pub(super) enum ArenaSparseNode { } impl ArenaSparseNode { - /// Returns the state of a Branch, Leaf, or Subtrie root node, or `None` for other types. + /// Returns the state of an `EmptyRoot`, `Branch`, `Leaf`, or `Subtrie` root node, or `None` for + /// other types. pub(super) fn state_ref(&self) -> Option<&ArenaSparseNodeState> { match self { + Self::EmptyRoot { state } | Self::Leaf { state, .. } => Some(state), Self::Branch(b) => Some(&b.state), - Self::Leaf { state, .. } => Some(state), Self::Subtrie(s) => s.arena[s.root].state_ref(), _ => None, } } - /// Returns a mutable reference to the state of a Branch or Leaf node. + /// Returns a mutable reference to the state of an `EmptyRoot`, `Branch`, or `Leaf` node. /// /// # Panics /// - /// Panics if called on a non-Branch/Leaf node. + /// Panics if called on a non-EmptyRoot/Branch/Leaf node. pub(super) fn state_mut(&mut self) -> &mut ArenaSparseNodeState { match self { + Self::EmptyRoot { state } | Self::Leaf { state, .. } => state, Self::Branch(b) => &mut b.state, - Self::Leaf { state, .. } => state, - _ => panic!("state_mut called on non-Branch/Leaf node"), + _ => panic!("state_mut called on non-EmptyRoot/Branch/Leaf node"), } } @@ -299,7 +315,7 @@ impl ArenaSparseNode { pub(super) fn from_proof_node(proof_node: ProofTrieNodeV2) -> Self { let ProofTrieNodeV2 { node, masks, .. } = proof_node; match node { - TrieNodeV2::EmptyRoot => Self::EmptyRoot, + TrieNodeV2::EmptyRoot => Self::EmptyRoot { state: ArenaSparseNodeState::Revealed }, TrieNodeV2::Leaf(leaf) => Self::Leaf { state: ArenaSparseNodeState::Revealed, key: leaf.key, diff --git a/crates/trie/sparse/src/state.rs b/crates/trie/sparse/src/state.rs index 321ab08d094..8d67c1157af 100644 --- a/crates/trie/sparse/src/state.rs +++ b/crates/trie/sparse/src/state.rs @@ -1,14 +1,14 @@ #[cfg(feature = "trie-debug")] use crate::debug_recorder::TrieDebugRecorder; -use crate::{traits::SparseTrie as SparseTrieTrait, ArenaParallelSparseTrie, RevealableSparseTrie}; +use crate::{ + traits::SparseTrie as SparseTrieTrait, ArenaParallelSparseTrie, RevealableSparseTrie, + TrieNodeEpoch, +}; use alloc::vec::Vec; use alloy_primitives::{map::B256Map, B256}; use either::Either; use reth_execution_errors::{SparseStateTrieResult, SparseTrieErrorKind}; -#[cfg(test)] -use reth_trie_common::prefix_set::TriePrefixSetsMut; use reth_trie_common::{ - prefix_set::{PrefixSet, TriePrefixSets}, updates::{StorageTrieUpdates, TrieUpdates}, DecodedMultiProof, MultiProof, Nibbles, ProofTrieNodeV2, }; @@ -353,15 +353,15 @@ where /// /// If the trie has not been revealed, this function does nothing. #[instrument(level = "debug", target = "trie::sparse", skip_all)] - pub fn calculate_subtries(&mut self) { + pub fn calculate_subtries(&mut self, new_epoch: TrieNodeEpoch) { if let RevealableSparseTrie::Revealed(trie) = &mut self.state { - trie.update_subtrie_hashes(); + trie.update_subtrie_hashes(new_epoch); } } /// Returns storage sparse trie root if the trie has been revealed. - pub fn storage_root(&mut self, account: &B256) -> Option { - self.storage.tries.get_mut(account).and_then(|trie| trie.root()) + pub fn storage_root(&mut self, account: &B256, new_epoch: TrieNodeEpoch) -> Option { + self.storage.tries.get_mut(account).and_then(|trie| trie.root(new_epoch)) } /// Returns mutable reference to the revealed account sparse trie. @@ -370,19 +370,22 @@ where } /// Returns sparse trie root. - pub fn root(&mut self) -> SparseStateTrieResult { + pub fn root(&mut self, new_epoch: TrieNodeEpoch) -> SparseStateTrieResult { // record revealed node metrics #[cfg(feature = "metrics")] self.metrics.record(); - Ok(self.revealed_trie_mut()?.root()) + Ok(self.revealed_trie_mut()?.root(new_epoch)) } /// Returns sparse trie root and trie updates. /// /// Returns an error if the account trie is still blind. #[instrument(level = "debug", target = "trie::sparse", skip_all)] - pub fn root_with_updates(&mut self) -> SparseStateTrieResult<(B256, TrieUpdates)> { + pub fn root_with_updates( + &mut self, + new_epoch: TrieNodeEpoch, + ) -> SparseStateTrieResult<(B256, TrieUpdates)> { // record revealed node metrics #[cfg(feature = "metrics")] self.metrics.record(); @@ -390,7 +393,7 @@ where let storage_tries = self.storage_trie_updates(); let revealed = self.revealed_trie_mut()?; - let (root, updates) = (revealed.root(), revealed.take_updates()); + let (root, updates) = (revealed.root(new_epoch), revealed.take_updates()); let updates = TrieUpdates { account_nodes: updates.updated_nodes, removed_nodes: updates.removed_nodes, @@ -455,14 +458,15 @@ where self.storage.tries.len() + self.storage.cleared_tries.len() } - /// Prunes account/storage tries according to the provided retained paths. All other revealed - /// paths are pruned to hash stubs or fully evicted. + /// Prunes account and storage trie nodes last modified before `prune_before`. + /// + /// Storage tries whose root epochs predate the cutoff are fully evicted. /// /// # Preconditions /// - /// All revealed account and storage tries must already have computed hashes via `root()` - /// / `storage_root()` for their current state. Pruning a dirty revealed trie is a hard - /// error and may panic. + /// Modified account and storage tries must already have computed hashes via `root()` / + /// `storage_root()` for their current state. Unmodified storage roots revealed only by + /// prewarming are treated as epoch zero. #[cfg(feature = "std")] #[instrument( level = "debug", @@ -470,17 +474,13 @@ where target = "trie::sparse", skip_all )] - pub fn prune(&mut self, retained_paths: TriePrefixSets) { - let retained_accounts = retained_paths.account_prefix_set.len(); - let retained_storage_tries = retained_paths.storage_prefix_sets.len(); + pub fn prune(&mut self, prune_before: TrieNodeEpoch) { let total_storage_tries_before = self.storage.tries.len(); - let TriePrefixSets { account_prefix_set, storage_prefix_sets, .. } = retained_paths; - let parent_span = tracing::Span::current(); let account_parent_span = parent_span.clone(); - // Prune account and storage tries in parallel using the same retained set. + // Prune account and storage tries in parallel using the same epoch cutoff. let (account_nodes_pruned, storage_tries_evicted) = rayon::join( || { let hashed_address = Option::::None; @@ -492,18 +492,14 @@ where ) .entered(); - self.state - .as_revealed_mut() - .map(|trie| trie.prune(account_prefix_set.slice())) - .unwrap_or(0) + self.state.as_revealed_mut().map(|trie| trie.prune(prune_before)).unwrap_or(0) }, - || self.storage.prune_by_retained_slots(storage_prefix_sets, &parent_span), + || self.storage.prune(prune_before, &parent_span), ); debug!( target: "trie::sparse", - retained_accounts, - retained_storage_tries, + prune_before = prune_before.get(), account_nodes_pruned, storage_tries_evicted, storage_tries_after = total_storage_tries_before - storage_tries_evicted, @@ -526,19 +522,14 @@ struct StorageTries { #[cfg(feature = "std")] impl StorageTries { - /// Prunes storage tries using the provided retained slots. - /// - /// Tries without retained slots are evicted entirely. Tries with retained slots are pruned to - /// those slots. - fn prune_by_retained_slots( - &mut self, - retained_slots: B256Map, - parent_span: &tracing::Span, - ) -> usize { - // Parallel pass: prune retained tries and clear evicted ones in place. - { - use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; - self.tries.par_iter_mut().for_each(|(address, trie)| { + /// Prunes storage tries by epoch, returning fully old tries to the reuse pool. + fn prune(&mut self, prune_before: TrieNodeEpoch, parent_span: &tracing::Span) -> usize { + use rayon::iter::{IntoParallelRefMutIterator, ParallelIterator}; + + let addresses_to_evict: Vec = self + .tries + .par_iter_mut() + .filter_map(|(address, trie)| { let hashed_address = Some(*address); let _span = tracing::trace_span!( target: "trie::sparse", @@ -548,28 +539,27 @@ impl StorageTries { ) .entered(); - if let Some(slots) = retained_slots.get(address) { - if let Some(t) = trie.as_revealed_mut() { - t.prune(slots.slice()); + let evict = trie.as_revealed_mut().is_none_or(|revealed| { + // Avoid traversing and compacting a trie that will be immediately evicted. + let root_epoch = + revealed.root_epoch().expect("storage trie root must not be dirty"); + if !root_epoch.should_prune(prune_before) { + revealed.prune(prune_before); } - } else { - trie.clear(); - } - }); - } + root_epoch.should_prune(prune_before) + }); - // Cheap sequential drain: move already-cleared tries into the reuse pool. - let addresses_to_evict: Vec = self - .tries - .keys() - .filter(|address| !retained_slots.contains_key(*address)) - .copied() + evict.then(|| { + trie.clear(); + *address + }) + }) .collect(); let evicted = addresses_to_evict.len(); self.cleared_tries.reserve(evicted); - for address in &addresses_to_evict { - if let Some(trie) = self.tries.remove(address) { + for address in addresses_to_evict { + if let Some(trie) = self.tries.remove(&address) { self.cleared_tries.push(trie); } } @@ -618,6 +608,10 @@ mod tests { TrieAccount, TrieMask, TrieNodeV2, }; + const fn epoch(value: u64) -> TrieNodeEpoch { + TrieNodeEpoch::new(value) + } + /// Create a leaf key (suffix) with given nibbles padded with zeros to reach `total_len`. fn leaf_key(suffix: impl AsRef<[u8]>, total_len: usize) -> Nibbles { let suffix = suffix.as_ref(); @@ -764,14 +758,17 @@ mod tests { } #[test] - fn prune_keeps_retained_paths_overlay_account_and_storage() { + fn prune_uses_epochs_for_account_and_storage_tries() { let mut sparse = SparseStateTrie::::default(); let account = B256::ZERO; - let hot_account = + let old_account = b256!("0x1000000000000000000000000000000000000000000000000000000000000000"); + let prewarmed_account = + b256!("0x2000000000000000000000000000000000000000000000000000000000000000"); let slot = B256::ZERO; let account_path = leaf_key([0x0], 64); + let old_account_path = leaf_key([0x1], 64); let storage_path = leaf_key([0x0], 64); let leaf_value = alloy_rlp::encode(TrieAccount::default()); @@ -811,7 +808,15 @@ mod tests { }, ), ( - hot_account, + old_account, + StorageMultiProof { + root: B256::ZERO, + subtree: subtree(), + branch_node_masks: Default::default(), + }, + ), + ( + prewarmed_account, StorageMultiProof { root: B256::ZERO, subtree: subtree(), @@ -823,8 +828,27 @@ mod tests { }; sparse.reveal_decoded_multiproof(multiproof.try_into().unwrap()).unwrap(); + + sparse.storage_root(&account, epoch(0)).unwrap(); + sparse.storage_root(&old_account, epoch(0)).unwrap(); + sparse.root(epoch(0)).unwrap(); + assert!(!sparse.storage_trie_ref(&prewarmed_account).unwrap().is_root_cached()); + + let mut storage_updates = B256Map::from_iter([( + slot, + LeafUpdate::Changed(alloy_rlp::encode_fixed_size(&U256::from(2)).to_vec()), + )]); + sparse + .storage_trie_mut(&account) + .unwrap() + .update_leaves(&mut storage_updates, |_, _| { + panic!("fully revealed storage trie must not request proofs") + }) + .unwrap(); + assert!(storage_updates.is_empty()); + let trie_account = TrieAccount { - storage_root: sparse.storage_root(&account).unwrap(), + storage_root: sparse.storage_root(&account, epoch(10)).unwrap(), ..Default::default() }; apply_account_update( @@ -832,15 +856,8 @@ mod tests { account, LeafUpdate::Changed(alloy_rlp::encode(trie_account)), ); - sparse.root().unwrap(); - let mut retained_paths = TriePrefixSetsMut::default(); - retained_paths.account_prefix_set.insert(Nibbles::unpack(account)); - retained_paths - .storage_prefix_sets - .entry(account) - .or_default() - .insert(Nibbles::unpack(slot)); - sparse.prune(retained_paths.freeze()); + let root_before = sparse.root(epoch(10)).unwrap(); + sparse.prune(epoch(10)); assert!(matches!( sparse.state_trie_ref().unwrap().find_leaf(&account_path, None), @@ -850,7 +867,19 @@ mod tests { sparse.storage_trie_ref(&account).unwrap().find_leaf(&storage_path, None), Ok(LeafLookup::Exists) )); - assert!(sparse.storage_trie_ref(&hot_account).is_none()); + assert!(matches!( + sparse.state_trie_ref().unwrap().find_leaf(&old_account_path, None), + Err(crate::LeafLookupError::BlindedNode { .. }) + )); + assert!(sparse.storage_trie_ref(&old_account).is_none()); + assert!(sparse.storage_trie_ref(&prewarmed_account).is_none()); + assert_eq!(sparse.root(epoch(10)).unwrap(), root_before); + + // An intermediate storage root does not advance without another change, so the trie + // becomes eligible once the cutoff passes its last modification epoch. + sparse.prune(epoch(11)); + assert!(sparse.storage_trie_ref(&account).is_none()); + assert_eq!(sparse.root(epoch(11)).unwrap(), root_before); } #[test] @@ -975,7 +1004,7 @@ mod tests { fn root_on_blind_trie_returns_blind_error() { let mut sparse = SparseStateTrie::::default(); - let err = sparse.root().unwrap_err(); + let err = sparse.root(epoch(0)).unwrap_err(); assert!(matches!(err.kind(), SparseStateTrieErrorKind::Sparse(SparseTrieErrorKind::Blind))); } @@ -1069,7 +1098,7 @@ mod tests { ) .unwrap(); - assert_eq!(sparse.root().unwrap(), root); + assert_eq!(sparse.root(epoch(0)).unwrap(), root); let address_3 = b256!("0x2000000000000000000000000000000000000000000000000000000000000000"); let account_3 = Account { nonce: account_1.nonce + 1, ..account_1 }; @@ -1089,7 +1118,7 @@ mod tests { .update_leaves(&mut updates, |_, _| {}) .unwrap(); assert!(updates.is_empty()); - trie_account_1.storage_root = sparse.storage_root(&address_1).unwrap(); + trie_account_1.storage_root = sparse.storage_root(&address_1, epoch(0)).unwrap(); apply_account_update( &mut sparse, address_1, @@ -1097,14 +1126,14 @@ mod tests { ); sparse.wipe_storage(address_2).unwrap(); - trie_account_2.storage_root = sparse.storage_root(&address_2).unwrap(); + trie_account_2.storage_root = sparse.storage_root(&address_2, epoch(0)).unwrap(); apply_account_update( &mut sparse, address_2, LeafUpdate::Changed(alloy_rlp::encode(trie_account_2)), ); - sparse.root().unwrap(); + sparse.root(epoch(0)).unwrap(); let sparse_updates = sparse.take_trie_updates().unwrap(); // TODO(alexey): assert against real state root calculation updates diff --git a/crates/trie/sparse/src/traits.rs b/crates/trie/sparse/src/traits.rs index b4bb8caa0f0..79a4a8d4972 100644 --- a/crates/trie/sparse/src/traits.rs +++ b/crates/trie/sparse/src/traits.rs @@ -16,6 +16,33 @@ use reth_trie_common::{ #[cfg(feature = "trie-debug")] use crate::debug_recorder::TrieDebugRecorder; +/// Modification epoch assigned to cached sparse trie nodes. +/// +/// Epochs must increase monotonically. Nodes materialized from the parent state without being +/// modified use [`Self::UNMODIFIED`]. +#[derive(Debug, Default, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub struct TrieNodeEpoch(u64); + +impl TrieNodeEpoch { + /// Epoch assigned to nodes materialized from the parent state without being modified. + pub const UNMODIFIED: Self = Self(0); + + /// Creates a new node modification epoch. + pub const fn new(epoch: u64) -> Self { + Self(epoch) + } + + /// Returns the inner epoch. + pub const fn get(self) -> u64 { + self.0 + } + + /// Returns whether a node with this epoch should be pruned at the provided cutoff. + pub const fn should_prune(self, prune_before: Self) -> bool { + self.0 < prune_before.0 + } +} + /// Describes an update to a leaf in the sparse trie. #[derive(Debug, Clone, PartialEq, Eq)] pub enum LeafUpdate { @@ -98,25 +125,28 @@ pub trait SparseTrie: Sized + Debug + Send + Sync { /// each node with [`TrieNodeV2::EmptyRoot`] to avoid cloning. fn reveal_nodes(&mut self, nodes: &mut [ProofTrieNodeV2]) -> SparseTrieResult<()>; - /// Calculates and returns the root hash of the trie. + /// Calculates and returns the root hash of the trie at the provided epoch. /// - /// This processes any dirty nodes by updating their RLP encodings - /// and returns the root hash. + /// This processes dirty nodes by updating their RLP encodings and caching their newest + /// modification at `new_epoch`, then returns the root hash. /// /// # Returns /// /// The root hash of the trie. - fn root(&mut self) -> B256; + fn root(&mut self, new_epoch: TrieNodeEpoch) -> B256; /// Returns true if the root node is cached and does not need any recomputation. fn is_root_cached(&self) -> bool; + /// Returns the root's modification epoch when it is clean, or `None` when it is dirty. + fn root_epoch(&self) -> Option; + /// Recalculates and updates the RLP hashes of subtries deeper than a certain level. The level /// is defined in the implementation. /// /// The root node is considered to be at level 0. This method is useful for optimizing /// hash recalculations after localized changes to the trie structure. - fn update_subtrie_hashes(&mut self); + fn update_subtrie_hashes(&mut self, new_epoch: TrieNodeEpoch); /// Retrieves a reference to the leaf value at the specified path. /// @@ -191,30 +221,17 @@ pub trait SparseTrie: Sized + Debug + Send + Sync { /// This is useful for reusing the trie without needing to reallocate memory. fn clear(&mut self); - /// Returns a cheap O(1) size hint for the trie representing the count of revealed - /// (non-Hash) nodes. - /// - /// This is used as a heuristic for prioritizing which storage tries to keep - /// during pruning. Larger values indicate larger tries that are more valuable to preserve. - fn size_hint(&self) -> usize; - - /// Prunes all subtrees that do not contain retained leaves. - /// - /// Each retained leaf is a full key path (usually 64 nibbles for hashed keys). - /// Any revealed subtree that is not a prefix of at least one retained key is collapsed into - /// hash stubs when hashes are available. + /// Collapses nodes last modified before `prune_before` into hash stubs. /// /// # Preconditions /// - /// Must be called only after `root()` has computed hashes for the current trie state. - /// Calling `prune` on a dirty trie is a hard error and may panic. - /// - /// `retained_leaves` must be sorted lexicographically. + /// The trie must not be dirty. An unmodified revealed root may be pruned because + /// proof-revealed descendants carry cached RLP nodes. /// /// # Returns /// /// The number of nodes converted to hash stubs. - fn prune(&mut self, retained_leaves: &[Nibbles]) -> usize; + fn prune(&mut self, prune_before: TrieNodeEpoch) -> usize; /// Takes the debug recorder out of this trie, replacing it with an empty one. /// diff --git a/crates/trie/sparse/src/trie.rs b/crates/trie/sparse/src/trie.rs index eb1292ed966..63ef6e1b81d 100644 --- a/crates/trie/sparse/src/trie.rs +++ b/crates/trie/sparse/src/trie.rs @@ -1,5 +1,6 @@ use crate::{ ArenaParallelSparseTrie, LeafUpdate, SparseTrie as SparseTrieTrait, SparseTrieUpdates, + TrieNodeEpoch, }; use alloc::{borrow::Cow, boxed::Box}; use alloy_primitives::{map::B256Map, B256}; @@ -178,8 +179,8 @@ impl RevealableSparseTrie { /// /// - `Some(B256)` with the calculated root hash if the trie is revealed. /// - `None` if the trie is still blind. - pub fn root(&mut self) -> Option { - Some(self.as_revealed_mut()?.root()) + pub fn root(&mut self, new_epoch: TrieNodeEpoch) -> Option { + Some(self.as_revealed_mut()?.root(new_epoch)) } /// Returns true if the root node is cached and does not need any recomputation. @@ -199,9 +200,12 @@ impl RevealableSparseTrie { /// - The trie root hash (`B256`). /// - A [`SparseTrieUpdates`] structure containing information about updated nodes. /// - `None` if the trie is still blind. - pub fn root_with_updates(&mut self) -> Option<(B256, SparseTrieUpdates)> { + pub fn root_with_updates( + &mut self, + new_epoch: TrieNodeEpoch, + ) -> Option<(B256, SparseTrieUpdates)> { let revealed = self.as_revealed_mut()?; - Some((revealed.root(), revealed.take_updates())) + Some((revealed.root(new_epoch), revealed.take_updates())) } /// Clears this trie, setting it to a blind state. diff --git a/crates/trie/sparse/tests/suite/lifecycle.rs b/crates/trie/sparse/tests/suite/lifecycle.rs index 3b30088fdb0..1cd8988952e 100644 --- a/crates/trie/sparse/tests/suite/lifecycle.rs +++ b/crates/trie/sparse/tests/suite/lifecycle.rs @@ -23,7 +23,7 @@ pub(super) fn test_full_lifecycle_update_root_take_updates(new_tr let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); // Cache initial hashes. - let _ = trie.root(); + let _ = trie.root(epoch(0)); // Insert 1 new leaf under the same hashed prefix and modify 1 existing leaf. let mut new_key = B256::ZERO; @@ -40,7 +40,7 @@ pub(super) fn test_full_lifecycle_update_root_take_updates(new_tr let mut leaf_updates = SuiteTestHarness::leaf_updates(&changeset); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let hash1 = trie.root(); + let hash1 = trie.root(epoch(0)); // Verify hash1 matches the reference trie with updated values. let mut expected_storage = storage; @@ -57,90 +57,10 @@ pub(super) fn test_full_lifecycle_update_root_take_updates(new_tr ); // Taking updates should not affect the cached root. - let hash2 = trie.root(); + let hash2 = trie.root(epoch(0)); assert_eq!(hash1, hash2, "root should be unchanged after take_updates"); } -/// Multiple rounds of (update → root → `take_updates`), followed by a prune, simulating block -/// processing. -pub(super) fn test_multi_round_update_take_updates_prune_cycle(new_trie: fn() -> T) { - // Build a trie with 10 primary leaves, each with a sibling under the same root child. - let mut storage: BTreeMap = BTreeMap::new(); - let mut keys = Vec::new(); - for i in 0u8..10 { - let mut key = B256::ZERO; - key.0[0] = i << 4; // nibble prefixes: 0x0, 0x1, 0x2, ... 0x9 - storage.insert(key, U256::from(i as u64 + 1)); - keys.push(key); - - let mut sibling = B256::ZERO; - sibling.0[0] = (i << 4) | 1; - storage.insert(sibling, U256::from(i as u64 + 100)); - } - - let mut harness = SuiteTestHarness::new(storage.clone()); - let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); - - // Cache initial hashes. - let _ = trie.root(); - - // --- Round 1: Update leaves A (keys[0]) and B (keys[1]) --- - let mut changeset1: BTreeMap = BTreeMap::new(); - changeset1.insert(keys[0], U256::from(100)); - changeset1.insert(keys[1], U256::from(200)); - let mut leaf_updates1 = SuiteTestHarness::leaf_updates(&changeset1); - harness.reveal_and_update(&mut trie, &mut leaf_updates1); - let root1 = trie.root(); - - // Verify root1 matches reference. - harness.apply_changeset(changeset1); - assert_eq!(root1, harness.original_root(), "round 1 root should match reference"); - - let _ = trie.take_updates(); - - // --- Round 2: Update leaves C (keys[2]) and D (keys[3]) --- - let mut changeset2: BTreeMap = BTreeMap::new(); - changeset2.insert(keys[2], U256::from(300)); - changeset2.insert(keys[3], U256::from(400)); - let mut leaf_updates2 = SuiteTestHarness::leaf_updates(&changeset2); - harness.reveal_and_update(&mut trie, &mut leaf_updates2); - let root2 = trie.root(); - - // Verify root2 matches reference. - harness.apply_changeset(changeset2); - assert_eq!(root2, harness.original_root(), "round 2 root should match reference"); - - let _ = trie.take_updates(); - - // --- Prune: retain only keys A,B,C,D --- - let retained: Vec = keys[0..4].iter().map(|k| Nibbles::unpack(*k)).collect(); - let pre_prune_size = trie.size_hint(); - trie.prune(&retained); - let post_prune_size = trie.size_hint(); - if pre_prune_size > 0 { - assert!(post_prune_size < pre_prune_size, "prune should reduce node count"); - } - - // Root should still be correct after prune. - let root_after_prune = trie.root(); - assert_eq!(root_after_prune, harness.original_root(), "root should be unchanged after prune"); - - // --- Round 3: Update leaf E (keys[4]) — needs re-reveal since it was pruned --- - let mut changeset3: BTreeMap = BTreeMap::new(); - changeset3.insert(keys[4], U256::from(500)); - let mut leaf_updates3 = SuiteTestHarness::leaf_updates(&changeset3); - harness.reveal_and_update(&mut trie, &mut leaf_updates3); - let root3 = trie.root(); - - // Verify root3 matches reference. - harness.apply_changeset(changeset3); - assert_eq!( - root3, - harness.original_root(), - "round 3 root should match reference after re-reveal" - ); -} - /// Core production loop: reveal → update → root. /// /// Build a 5-leaf trie, reveal all proofs, apply 2 modifications + 1 removal, @@ -167,7 +87,7 @@ pub(super) fn test_reveal_update_root_basic_lifecycle(new_trie: f let mut leaf_updates = SuiteTestHarness::leaf_updates(&changeset); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let root = trie.root(); + let root = trie.root(epoch(0)); // Build reference trie with same mutations. let mut expected_storage = storage; @@ -255,7 +175,7 @@ pub(super) fn test_incremental_reveal_and_update_with_retry(new_t expected_storage.insert(group_b_keys[1], U256::from(900)); let expected_harness = SuiteTestHarness::new(expected_storage); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!( root, expected_harness.original_root(), @@ -266,8 +186,7 @@ pub(super) fn test_incremental_reveal_and_update_with_retry(new_t /// End-to-end block processing with storage + account coordination. /// /// Simulates a complete block processing cycle: receive state updates → apply to storage -/// tries → compute storage roots → promote to account trie → compute state root → take -/// updates → commit → prune for next block. +/// tries → compute storage roots → promote to account trie → compute state root → take updates. pub(super) fn test_full_block_processing_lifecycle(new_trie: fn() -> T) { // --- Setup: Build account trie with 5 accounts --- // A1 storage: 5 slots, A2 storage: 3 slots, A3-A5: empty storage. @@ -322,9 +241,9 @@ pub(super) fn test_full_block_processing_lifecycle(new_trie: fn() let mut acct_trie: T = acct_harness.init_trie_fully_revealed(true, new_trie); // Cache initial hashes for all tries. - let _ = a1_trie.root(); - let _ = a2_trie.root(); - let _ = acct_trie.root(); + let _ = a1_trie.root(epoch(0)); + let _ = a2_trie.root(epoch(0)); + let _ = acct_trie.root(epoch(0)); // --- Storage phase --- // A1: modify S1 (key 0x10), remove S2 (key 0x20), add S6 (key 0x60). @@ -351,8 +270,8 @@ pub(super) fn test_full_block_processing_lifecycle(new_trie: fn() a2_harness.reveal_and_update(&mut a2_trie, &mut a2_leaf_updates); // --- Storage root phase --- - let sr1 = a1_trie.root(); - let sr2 = a2_trie.root(); + let sr1 = a1_trie.root(epoch(0)); + let sr2 = a2_trie.root(epoch(0)); // Verify storage roots match references. a1_harness.apply_changeset(a1_changeset); @@ -371,7 +290,7 @@ pub(super) fn test_full_block_processing_lifecycle(new_trie: fn() acct_harness.reveal_and_update(&mut acct_trie, &mut acct_leaf_updates); // --- State root phase --- - let state_root = acct_trie.root(); + let state_root = acct_trie.root(epoch(0)); // Verify state root matches reference. acct_harness.apply_changeset(acct_changeset); @@ -381,14 +300,6 @@ pub(super) fn test_full_block_processing_lifecycle(new_trie: fn() let _ = acct_trie.take_updates(); let _ = a1_trie.take_updates(); let _ = a2_trie.take_updates(); - - // Prune account trie, retaining only A1 and A2 paths. - let retained: Vec = acct_keys[0..2].iter().map(|k| Nibbles::unpack(*k)).collect(); - acct_trie.prune(&retained); - - // Post-prune root should still be state_root. - let post_prune_root = acct_trie.root(); - assert_eq!(post_prune_root, state_root, "root should be unchanged after prune"); } /// Prewarm via `Touched`, then mutate via `Changed`. @@ -442,7 +353,7 @@ pub(super) fn test_touched_prewarm_then_changed_update(new_trie: harness.reveal_and_update(&mut trie, &mut leaf_updates); // Step 3: Compute root and verify against reference. - let root = trie.root(); + let root = trie.root(epoch(0)); harness.apply_changeset(changeset); assert_eq!( @@ -513,7 +424,7 @@ pub(super) fn test_touched_on_blinded_triggers_proof_then_changed_succeeds(new_tri trie.update_leaves(&mut leaf_updates, |_, _| {}).expect("update_leaves should succeed"); // Step 5: Compute root and verify against reference. - let root = trie.root(); + let root = trie.root(epoch(0)); let mut changeset = BTreeMap::new(); changeset.insert(key1, new_value); @@ -620,7 +531,7 @@ pub(super) fn test_find_leaf_before_update_to_check_existence(new trie.update_leaves(&mut leaf_updates, |_, _| {}).expect("update_leaves should succeed"); // Step 4: Compute root and verify against reference. - let root = trie.root(); + let root = trie.root(epoch(0)); let mut changeset = BTreeMap::new(); changeset.insert(key2, new_key2_value); @@ -636,10 +547,10 @@ pub(super) fn test_find_leaf_before_update_to_check_existence(new ); } -/// After prune, hot leaves update immediately; cold leaves need re-reveal. +/// After epoch pruning, recently changed leaves update immediately and old leaves need re-reveal. /// -/// Build 10-leaf trie, do Block 1 (update K1,K2,K3 → commit → prune retaining K1,K2), -/// then Block 2: update K1 (hot, works immediately), update K5 (cold, needs re-reveal). +/// Build a trie at epoch 0, update K1,K2,K3 at epoch 1, prune before epoch 1, then update +/// K1 without a proof and K5 after re-revealing it. pub(super) fn test_prune_then_reuse_for_next_block(new_trie: fn() -> T) { // Build a trie with 10 leaves. let mut storage: BTreeMap = BTreeMap::new(); @@ -655,7 +566,7 @@ pub(super) fn test_prune_then_reuse_for_next_block(new_trie: fn() let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); // Cache initial hashes. - let _ = trie.root(); + let _ = trie.root(epoch(0)); // --- Block 1: Update K1 (keys[0]), K2 (keys[1]), K3 (keys[2]) --- let mut changeset1: BTreeMap = BTreeMap::new(); @@ -664,23 +575,25 @@ pub(super) fn test_prune_then_reuse_for_next_block(new_trie: fn() changeset1.insert(keys[2], U256::from(300)); let mut leaf_updates1 = SuiteTestHarness::leaf_updates(&changeset1); harness.reveal_and_update(&mut trie, &mut leaf_updates1); - let root1 = trie.root(); + let root1 = trie.root(epoch(1)); harness.apply_changeset(changeset1); assert_eq!(root1, harness.original_root(), "block 1 root should match reference"); let _ = trie.take_updates(); - // --- Prune: retain only K1 and K2 as "hot" --- - let retained: Vec = [keys[0], keys[1]].iter().map(|k| Nibbles::unpack(*k)).collect(); - trie.prune(&retained); + trie.prune(epoch(1)); // --- Block 2 — hot path: Update K1 with a new value --- let mut changeset_hot: BTreeMap = BTreeMap::new(); changeset_hot.insert(keys[0], U256::from(999)); let mut leaf_updates_hot = SuiteTestHarness::leaf_updates(&changeset_hot); - harness.reveal_and_update(&mut trie, &mut leaf_updates_hot); - let root_hot = trie.root(); + trie.update_leaves(&mut leaf_updates_hot, |_, _| { + panic!("leaf changed at the cutoff epoch must remain revealed") + }) + .expect("update_leaves should succeed"); + assert!(leaf_updates_hot.is_empty()); + let root_hot = trie.root(epoch(2)); harness.apply_changeset(changeset_hot); assert_eq!( @@ -693,8 +606,12 @@ pub(super) fn test_prune_then_reuse_for_next_block(new_trie: fn() let mut changeset_cold: BTreeMap = BTreeMap::new(); changeset_cold.insert(keys[4], U256::from(555)); let mut leaf_updates_cold = SuiteTestHarness::leaf_updates(&changeset_cold); + let mut proof_requested = false; + trie.update_leaves(&mut leaf_updates_cold, |_, _| proof_requested = true) + .expect("update_leaves should succeed"); + assert!(proof_requested, "leaf older than the cutoff should require a proof"); harness.reveal_and_update(&mut trie, &mut leaf_updates_cold); - let root_cold = trie.root(); + let root_cold = trie.root(epoch(2)); harness.apply_changeset(changeset_cold); assert_eq!( diff --git a/crates/trie/sparse/tests/suite/main.rs b/crates/trie/sparse/tests/suite/main.rs index 27348f3787d..4666d5b0c85 100644 --- a/crates/trie/sparse/tests/suite/main.rs +++ b/crates/trie/sparse/tests/suite/main.rs @@ -15,15 +15,14 @@ //! - [`wipe_clear`]: Tests for `wipe` and `clear` //! - [`get_leaf_value`]: Tests for `get_leaf_value` //! - [`find_leaf`]: Tests for `find_leaf` -//! - [`size_hint`]: Tests for `size_hint` //! - [`lifecycle`]: Integration tests exercising multiple methods together use alloy_primitives::{map::B256Map, B256, U256}; use alloy_rlp::{encode_fixed_size, Decodable}; use alloy_trie::EMPTY_ROOT_HASH; use reth_trie::test_utils::TrieTestHarness; -use reth_trie_common::{Nibbles, ProofV2Target, ProofV2TargetParent, TrieNodeV2}; -use reth_trie_sparse::{LeafLookup, LeafLookupError, LeafUpdate, SparseTrie}; +use reth_trie_common::{Nibbles, ProofV2Target, TrieNodeV2}; +use reth_trie_sparse::{LeafLookup, LeafLookupError, LeafUpdate, SparseTrie, TrieNodeEpoch}; use std::{collections::BTreeMap, iter::once}; mod find_leaf; @@ -33,11 +32,14 @@ mod prune; mod reveal_nodes; mod root; mod set_root; -mod size_hint; mod take_updates; mod update_leaves; mod wipe_clear; +const fn epoch(value: u64) -> TrieNodeEpoch { + TrieNodeEpoch::new(value) +} + // --------------------------------------------------------------------------- // Test harness // --------------------------------------------------------------------------- @@ -196,7 +198,6 @@ use prune::*; use reveal_nodes::*; use root::*; use set_root::*; -use size_hint::*; use take_updates::*; use update_leaves::*; use wipe_clear::*; @@ -274,13 +275,9 @@ sparse_trie_tests! { // prune - test_prune_retains_specified_leaves, - test_prune_keeps_upper_children_of_retained_branch, - test_prune_keeps_lower_children_of_retained_branch, - test_prune_protects_children_by_parent_base_path, + test_prune_retains_recent_leaves, + test_prune_retains_structurally_modified_branch, test_prune_reduces_node_count, - test_prune_empty_retained_set, - test_prune_requires_computed_hashes, test_prune_then_update_and_recompute_root, test_prune_then_reveal_pruned_subtree, test_prune_mixed_embedded_and_hashed_nodes, @@ -307,12 +304,8 @@ sparse_trie_tests! { test_find_leaf_nonexistent_extension_divergence, test_find_leaf_nonexistent_leaf_divergence, - // size_hint - test_size_hint_reflects_leaf_count, - // lifecycle (integration tests) test_full_lifecycle_update_root_take_updates, - test_multi_round_update_take_updates_prune_cycle, test_reveal_update_root_basic_lifecycle, test_incremental_reveal_and_update_with_retry, test_full_block_processing_lifecycle, diff --git a/crates/trie/sparse/tests/suite/prune.rs b/crates/trie/sparse/tests/suite/prune.rs index 1954add1a45..624b3b41cd2 100644 --- a/crates/trie/sparse/tests/suite/prune.rs +++ b/crates/trie/sparse/tests/suite/prune.rs @@ -1,49 +1,6 @@ use super::*; -use alloy_trie::{nodes::BranchNode, TrieMask}; -use reth_trie_common::{BranchNodeV2, LeafNode, ProofTrieNodeV2, RlpNode}; - -fn key_with_prefix(prefix: &[u8]) -> B256 { - let mut key = B256::ZERO; - for (idx, &nibble) in prefix.iter().enumerate() { - let byte = &mut key.0[idx / 2]; - if idx % 2 == 0 { - *byte |= nibble << 4; - } else { - *byte |= nibble; - } - } - key -} -fn changed_update(value: u64) -> LeafUpdate { - LeafUpdate::Changed(encode_fixed_size(&U256::from(value)).to_vec()) -} - -fn assert_update_requests_parent( - trie: &mut T, - key: B256, - parent: ProofV2TargetParent, - value: u64, -) { - let mut leaf_updates = B256Map::from_iter([(key, changed_update(value))]); - let mut targets = Vec::new(); - trie.update_leaves(&mut leaf_updates, |key, parent| { - targets.push((key, parent)); - }) - .expect("update_leaves should succeed"); - - assert_eq!(targets, vec![(key, parent)]); - assert!( - leaf_updates.contains_key(&key), - "update should remain pending until proof is revealed" - ); -} - -fn rlp_node(node: TrieNodeV2) -> RlpNode { - RlpNode::from_rlp(&alloy_rlp::encode(node)) -} - -pub(super) fn test_prune_retains_specified_leaves(new_trie: fn() -> T) { +pub(super) fn test_prune_retains_recent_leaves(new_trie: fn() -> T) { let mut key_a = B256::ZERO; key_a.0[0] = 0x10; let mut key_b = B256::ZERO; @@ -66,130 +23,89 @@ pub(super) fn test_prune_retains_specified_leaves(new_trie: fn() let harness = SuiteTestHarness::new(storage); let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - // Compute root before prune. - let hash1 = trie.root(); - - // Retain leaves A and B, prune the rest. - let retained = [Nibbles::unpack(key_a), Nibbles::unpack(key_b)]; - trie.prune(&retained); - - // Root must be unchanged after prune. - let hash2 = trie.root(); - assert_eq!(hash1, hash2, "root hash should be unchanged after prune"); - - // Retained leaves must still be accessible. - let val_a = trie.get_leaf_value(&Nibbles::unpack(key_a)); - assert!(val_a.is_some(), "retained leaf A should be accessible after prune"); - - let val_b = trie.get_leaf_value(&Nibbles::unpack(key_b)); - assert!(val_b.is_some(), "retained leaf B should be accessible after prune"); -} - -pub(super) fn test_prune_keeps_upper_children_of_retained_branch( - new_trie: fn() -> T, -) { - let retained_key = key_with_prefix(&[0x0]); - let protected_key_a = key_with_prefix(&[0x1, 0x2]); - let protected_key_b = key_with_prefix(&[0x1, 0x3]); - let storage = BTreeMap::from([ - (retained_key, U256::from(1)), - (protected_key_a, U256::from(2)), - (protected_key_b, U256::from(3)), - ]); + let initial_root = trie.root(epoch(0)); + assert_eq!(trie.prune(epoch(0)), 0, "the epoch cutoff must be strict"); + assert_eq!(trie.root(epoch(0)), initial_root); - let harness = SuiteTestHarness::new(storage); - let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - let root_before = trie.root(); + let mut updates = SuiteTestHarness::leaf_updates(&BTreeMap::from([ + (key_a, U256::from(10)), + (key_b, U256::from(20)), + ])); + trie.update_leaves(&mut updates, |_, _| panic!("fully revealed trie must not request proofs")) + .expect("update_leaves should succeed"); + assert!(updates.is_empty()); - let retained = [Nibbles::unpack(retained_key)]; - let pruned = trie.prune(&retained); + let root_before = trie.root(epoch(2)); + assert!(trie.prune(epoch(2)) > 0); + assert_eq!(trie.root(epoch(3)), root_before, "root hash should be unchanged after prune"); - assert_eq!(trie.root(), root_before, "root must not change after prune"); - assert_eq!(pruned, 2, "protected branch children should be blinded, not removed"); - assert_update_requests_parent(&mut trie, protected_key_a, ProofV2TargetParent::new(1), 20); + assert!( + trie.get_leaf_value(&Nibbles::unpack(key_a)).is_some(), + "leaf at the cutoff epoch should remain accessible" + ); + assert!( + trie.get_leaf_value(&Nibbles::unpack(key_b)).is_some(), + "leaf at the cutoff epoch should remain accessible" + ); + assert!( + trie.get_leaf_value(&Nibbles::unpack(key_c)).is_none(), + "leaf older than the cutoff should be blinded" + ); } -pub(super) fn test_prune_keeps_lower_children_of_retained_branch( - new_trie: fn() -> T, -) { - let retained_key = key_with_prefix(&[0x0, 0x0, 0x0]); - let protected_key_a = key_with_prefix(&[0x0, 0x0, 0x1, 0x2]); - let protected_key_b = key_with_prefix(&[0x0, 0x0, 0x1, 0x3]); - let storage = BTreeMap::from([ - (retained_key, U256::from(1)), - (protected_key_a, U256::from(2)), - (protected_key_b, U256::from(3)), +pub(super) fn test_prune_retains_structurally_modified_branch(new_trie: fn() -> T) { + let key = |first_byte, second_byte| { + let mut key = B256::ZERO; + key.0[0] = first_byte; + key.0[1] = second_byte; + key + }; + let old_keys = [key(0x00, 0x00), key(0x00, 0x10), key(0x00, 0x20)]; + let sibling = key(0x01, 0x00); + let mut storage = BTreeMap::from([ + (old_keys[0], U256::from(1)), + (old_keys[1], U256::from(2)), + (old_keys[2], U256::from(3)), + (sibling, U256::from(4)), ]); - let harness = SuiteTestHarness::new(storage); - let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - let root_before = trie.root(); - - let retained = [Nibbles::unpack(retained_key)]; - let pruned = trie.prune(&retained); - - assert_eq!(trie.root(), root_before, "root must not change after prune"); - assert_eq!(pruned, 2, "protected lower branch children should be blinded, not removed"); - assert_update_requests_parent(&mut trie, protected_key_a, ProofV2TargetParent::new(3), 20); -} - -pub(super) fn test_prune_protects_children_by_parent_base_path(new_trie: fn() -> T) { - let protected_leaf_a = TrieNodeV2::Leaf(LeafNode::new(Nibbles::default(), vec![0x32])); - let protected_leaf_b = TrieNodeV2::Leaf(LeafNode::new(Nibbles::default(), vec![0x33])); - - let protected_state_mask = TrieMask::new(0b1100); - let protected_stack = - vec![rlp_node(protected_leaf_a.clone()), rlp_node(protected_leaf_b.clone())]; - let protected_branch_rlp = RlpNode::from_rlp(&alloy_rlp::encode(BranchNode::new( - protected_stack.clone(), - protected_state_mask, - ))); - let retained_parent = TrieNodeV2::Branch(BranchNodeV2::new( - Nibbles::from_nibbles([0xa]), - protected_stack, - protected_state_mask, - Some(protected_branch_rlp), - )); - let root = TrieNodeV2::Branch(BranchNodeV2::new( - Nibbles::default(), - vec![rlp_node(retained_parent.clone())], - TrieMask::new(0b0001), - None, - )); - let mut trie = (new_trie)(); - trie.set_root(root, None, false).expect("set_root should succeed"); - trie.reveal_nodes(&mut [ - ProofTrieNodeV2 { path: Nibbles::from_nibbles([0x0]), node: retained_parent, masks: None }, - ProofTrieNodeV2 { - path: Nibbles::from_nibbles([0x0, 0xa, 0x2]), - node: protected_leaf_a, - masks: None, - }, - ProofTrieNodeV2 { - path: Nibbles::from_nibbles([0x0, 0xa, 0x3]), - node: protected_leaf_b, - masks: None, - }, - ]) - .expect("reveal_nodes should succeed"); - - let root_before = trie.root(); - let retained = [Nibbles::from_nibbles([0x0, 0x0])]; - let pruned = trie.prune(&retained); - - assert_eq!(pruned, 0); - assert_eq!(trie.root(), root_before, "root must not change after prune"); + let mut initial_updates = SuiteTestHarness::leaf_updates(&storage); + trie.update_leaves(&mut initial_updates, |_, _| panic!("empty trie must not request proofs")) + .expect("update_leaves should succeed"); + assert!(initial_updates.is_empty()); + trie.root(epoch(10)); + + let mut deletion = SuiteTestHarness::leaf_updates(&BTreeMap::from([(old_keys[2], U256::ZERO)])); + trie.update_leaves(&mut deletion, |_, _| panic!("fully revealed trie must not request proofs")) + .expect("update_leaves should succeed"); + assert!(deletion.is_empty()); + storage.remove(&old_keys[2]); + + let root_before = trie.root(epoch(20)); + assert_eq!(root_before, SuiteTestHarness::new(storage.clone()).original_root()); + trie.prune(epoch(20)); + assert_eq!(trie.root(epoch(20)), root_before, "root hash should be unchanged after prune"); + + let inserted = key(0x00, 0x30); + let mut insertion = + SuiteTestHarness::leaf_updates(&BTreeMap::from([(inserted, U256::from(5))])); + trie.update_leaves(&mut insertion, |_, _| { + panic!("structurally modified branch at the cutoff epoch must remain revealed") + }) + .expect("update_leaves should succeed"); + assert!(insertion.is_empty()); + + storage.insert(inserted, U256::from(5)); + assert_eq!(trie.root(epoch(21)), SuiteTestHarness::new(storage).original_root()); } /// Pruning should reduce the node count. /// -/// Build a trie with several root children that each contain grandchildren, fully reveal -/// and compute root. Then prune retaining only 1 leaf. `size_hint()` must -/// decrease and `prune` must return > 0. +/// Build a trie with several root children that each contain grandchildren, fully reveal, +/// compute the root, and prune all nodes older than epoch 1. pub(super) fn test_prune_reduces_node_count(new_trie: fn() -> T) { - // Create 16 pairs with different first nibbles. Pruning keeps direct - // children of the retained root branch, so each child needs grandchildren that can be pruned. + // Create 16 pairs with different first nibbles so pruning exercises nested branches. let keys: Vec = (0u8..16) .flat_map(|i| { [0u8, 1].map(move |child| { @@ -207,92 +123,12 @@ pub(super) fn test_prune_reduces_node_count(new_trie: fn() -> T) let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); // Compute root to cache hashes (required for pruning). - let _root = trie.root(); - - let size_before = trie.size_hint(); + let root_before = trie.root(epoch(0)); - // Retain only the first leaf. - let retained = [Nibbles::unpack(keys[0])]; - let pruned_count = trie.prune(&retained); - - let size_after = trie.size_hint(); + let pruned_count = trie.prune(epoch(1)); + assert_eq!(trie.root(epoch(1)), root_before, "root hash should be unchanged after prune"); assert!(pruned_count > 0, "prune should convert at least one node to a stub"); - assert!( - size_after < size_before, - "size_hint should decrease after prune: before={size_before}, after={size_after}" - ); -} - -/// Pruning with an empty retained set should convert all subtrees to -/// hash stubs (maximum pruning). Root hash must be unchanged. -pub(super) fn test_prune_empty_retained_set(new_trie: fn() -> T) { - let keys: Vec = (0u8..16) - .map(|i| { - let mut k = B256::ZERO; - k.0[0] = (i + 1) << 4; - k - }) - .collect(); - - let storage: BTreeMap = - keys.iter().enumerate().map(|(i, k)| (*k, U256::from(i + 1))).collect(); - - let harness = SuiteTestHarness::new(storage); - let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - - let hash_before = trie.root(); - - let size_before = trie.size_hint(); - - // Prune with empty retained set — maximum pruning. - let pruned_count = trie.prune(&[]); - - let hash_after = trie.root(); - let size_after = trie.size_hint(); - - assert_eq!(hash_before, hash_after, "root hash should be unchanged after prune"); - assert!(pruned_count > 0, "prune should convert at least one node to a stub"); - assert!( - size_after < size_before, - "size_hint should decrease after max prune: before={size_before}, after={size_after}" - ); -} - -pub(super) fn test_prune_requires_computed_hashes(new_trie: fn() -> T) { - let keys: Vec = (0u8..5) - .map(|i| { - let mut k = B256::ZERO; - k.0[0] = (i + 1) << 4; - k - }) - .collect(); - - let storage: BTreeMap = - keys.iter().enumerate().map(|(i, k)| (*k, U256::from(i + 1))).collect(); - - let harness = SuiteTestHarness::new(storage); - let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - - // Dirty the trie by updating a leaf — do NOT call root() to compute hashes. - let mut leaf_updates: B256Map = B256Map::default(); - leaf_updates.insert(keys[0], LeafUpdate::Changed(encode_fixed_size(&U256::from(999)).to_vec())); - trie.update_leaves(&mut leaf_updates, |_, _| {}).expect("update_leaves should succeed"); - - // Prune without having called root() — dirty nodes lack cached hashes. - let retained = vec![Nibbles::unpack(keys[0])]; - let pruned_count = trie.prune(&retained); - - // Compare against pruning after root() is called (clean state). - // With dirty nodes, pruning is limited because dirty subtrees lack cached hashes. - let mut trie_clean: T = harness.init_trie_fully_revealed(false, new_trie); - trie_clean.root(); - let clean_pruned = trie_clean.prune(&retained); - - assert!( - pruned_count <= clean_pruned, - "dirty prune ({pruned_count}) should not exceed clean prune ({clean_pruned})" - ); } pub(super) fn test_prune_then_update_and_recompute_root(new_trie: fn() -> T) { @@ -310,17 +146,32 @@ pub(super) fn test_prune_then_update_and_recompute_root(new_trie: let harness = SuiteTestHarness::new(storage.clone()); let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - trie.root(); + trie.root(epoch(0)); - let retained = vec![Nibbles::unpack(keys[0]), Nibbles::unpack(keys[1])]; - trie.prune(&retained); + let mut first_update = B256Map::from_iter([( + keys[0], + LeafUpdate::Changed(encode_fixed_size(&U256::from(100)).to_vec()), + )]); + trie.update_leaves(&mut first_update, |_, _| { + panic!("fully revealed trie must not request proofs") + }) + .expect("update_leaves should succeed"); + assert!(first_update.is_empty()); + trie.root(epoch(2)); + trie.prune(epoch(2)); let new_value = U256::from(999); - let mut leaf_updates: B256Map = B256Map::default(); - leaf_updates.insert(keys[0], LeafUpdate::Changed(encode_fixed_size(&new_value).to_vec())); - harness.reveal_and_update(&mut trie, &mut leaf_updates); + let mut leaf_updates = B256Map::from_iter([( + keys[0], + LeafUpdate::Changed(encode_fixed_size(&new_value).to_vec()), + )]); + trie.update_leaves(&mut leaf_updates, |_, _| { + panic!("leaf at the cutoff epoch should remain revealed") + }) + .expect("update_leaves should succeed"); + assert!(leaf_updates.is_empty()); - let root_after = trie.root(); + let root_after = trie.root(epoch(3)); let mut expected_storage = storage; expected_storage.insert(keys[0], new_value); @@ -345,17 +196,15 @@ pub(super) fn test_prune_then_reveal_pruned_subtree(new_trie: fn( let harness = SuiteTestHarness::new(storage.clone()); let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - trie.root(); - - let retained = vec![Nibbles::unpack(keys[0])]; - trie.prune(&retained); + trie.root(epoch(0)); + trie.prune(epoch(1)); let new_value = U256::from(777); let mut leaf_updates: B256Map = B256Map::default(); leaf_updates.insert(keys[2], LeafUpdate::Changed(encode_fixed_size(&new_value).to_vec())); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let root_after = trie.root(); + let root_after = trie.root(epoch(2)); let mut expected_storage = storage; expected_storage.insert(keys[2], new_value); @@ -393,9 +242,9 @@ pub(super) fn test_prune_mixed_embedded_and_hashed_nodes(new_trie }) .expect("update_leaves should succeed"); - let root_before = trie.root(); - trie.prune(&[]); - let root_after = trie.root(); + let root_before = trie.root(epoch(0)); + trie.prune(epoch(1)); + let root_after = trie.root(epoch(1)); assert_eq!(root_before, root_after, "root hash must be preserved after pruning mixed trie"); } @@ -416,12 +265,12 @@ pub(super) fn test_prune_then_update_no_panic(new_trie: fn() -> T let harness = SuiteTestHarness::new(storage.clone()); let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - let root_before_prune = trie.root(); + let root_before_prune = trie.root(epoch(0)); - // Prune everything. - trie.prune(&[]); + // Prune everything older than epoch 1. + trie.prune(epoch(1)); - let hash1 = trie.root(); + let hash1 = trie.root(epoch(1)); assert_eq!(hash1, root_before_prune, "root after prune must equal root before prune"); // Insert a brand-new key not previously in the trie. @@ -434,7 +283,7 @@ pub(super) fn test_prune_then_update_no_panic(new_trie: fn() -> T // The update will hit blinded nodes — the reveal_and_update loop supplies proofs. harness.reveal_and_update(&mut trie, &mut leaf_updates); - let hash2 = trie.root(); + let hash2 = trie.root(epoch(2)); assert_ne!(hash2, hash1, "root should change after inserting a new leaf"); } @@ -447,13 +296,14 @@ pub(super) fn test_prune_only_descends_into_branch_root(new_trie: let harness = SuiteTestHarness::new(storage); let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - let _root = trie.root(); - let pruned = trie.prune(&[]); + let _root = trie.root(epoch(0)); + let pruned = trie.prune(epoch(1)); assert_eq!(pruned, 0, "non-branch root should not prune any nodes"); // Empty root: also not a branch. let mut empty_trie = (new_trie)(); - let pruned_empty = empty_trie.prune(&[]); + empty_trie.root(epoch(0)); + let pruned_empty = empty_trie.prune(epoch(1)); assert_eq!(pruned_empty, 0, "empty root should not prune any nodes"); } @@ -466,6 +316,8 @@ pub(super) fn test_prune_handles_small_subtrie_root_nodes(new_tri // - Group A (nibble 0x1): 16 leaves with large values → hashable subtrie root (RLP ≥ 32 bytes) // - Group B (nibble 0x2): 1 leaf with a small value → small subtrie root (RLP < 32 bytes) let mut storage = BTreeMap::new(); + let mut large_key = B256::ZERO; + large_key.0[0] = 0x10; for i in 0u8..16 { let mut key = B256::ZERO; key.0[0] = 0x10 | (i & 0x0F); @@ -480,14 +332,23 @@ pub(super) fn test_prune_handles_small_subtrie_root_nodes(new_tri let harness = SuiteTestHarness::new(storage); let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - let root_before = trie.root(); - assert_eq!(root_before, harness.original_root()); + trie.root(epoch(0)); + let mut update = B256Map::from_iter([( + small_key, + LeafUpdate::Changed(encode_fixed_size(&U256::from(2)).to_vec()), + )]); + trie.update_leaves(&mut update, |_, _| panic!("fully revealed trie must not request proofs")) + .expect("update_leaves should succeed"); + assert!(update.is_empty()); - // Prune retaining only the small-subtrie leaf — the large subtrie should - // be replaced by hash stubs, and the small subtrie handled gracefully. - let retained = vec![Nibbles::unpack(small_key)]; - trie.prune(&retained); + let root_before = trie.root(epoch(2)); + trie.prune(epoch(2)); - let root_after = trie.root(); + let root_after = trie.root(epoch(2)); assert_eq!(root_after, root_before, "root must not change after prune"); + assert!(trie.get_leaf_value(&Nibbles::unpack(small_key)).is_some()); + assert!( + trie.get_leaf_value(&Nibbles::unpack(large_key)).is_none(), + "old large subtrie should be blinded" + ); } diff --git a/crates/trie/sparse/tests/suite/reveal_nodes.rs b/crates/trie/sparse/tests/suite/reveal_nodes.rs index 0f6733cf5aa..8da0e3c9fdf 100644 --- a/crates/trie/sparse/tests/suite/reveal_nodes.rs +++ b/crates/trie/sparse/tests/suite/reveal_nodes.rs @@ -20,12 +20,12 @@ pub(super) fn test_reveal_nodes_empty_slice(new_trie: fn() -> T) let mut trie = (new_trie)(); trie.set_root(root_node.node, root_node.masks, true).expect("set_root should succeed"); - let root_before = trie.root(); + let root_before = trie.root(epoch(0)); // Call reveal_nodes with an empty slice — should be a no-op. trie.reveal_nodes(&mut []).expect("reveal_nodes with empty slice should succeed"); - let root_after = trie.root(); + let root_after = trie.root(epoch(0)); assert_eq!(root_before, root_after, "root should be unchanged after empty reveal_nodes"); } @@ -47,7 +47,7 @@ pub(super) fn test_reveal_nodes_single_leaf(new_trie: fn() -> T) // Set root and reveal only one leaf's proof. let mut trie: T = harness.init_trie_with_targets(&[key_a], true, new_trie); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!(root, harness.original_root()); } @@ -69,7 +69,7 @@ pub(super) fn test_reveal_nodes_idempotent(new_trie: fn() -> T) { // First reveal: set root and reveal all proof nodes. let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); - let root_first = trie.root(); + let root_first = trie.root(epoch(0)); assert_eq!(root_first, harness.original_root()); // Second reveal: reveal the same proof nodes again. @@ -78,7 +78,7 @@ pub(super) fn test_reveal_nodes_idempotent(new_trie: fn() -> T) { let (mut proof_nodes, _) = harness.proof_v2(&mut targets); trie.reveal_nodes(&mut proof_nodes).expect("second reveal_nodes should succeed"); - let root_second = trie.root(); + let root_second = trie.root(epoch(0)); assert_eq!(root_first, root_second, "root should be unchanged after double reveal"); } @@ -104,7 +104,7 @@ pub(super) fn test_reveal_nodes_with_branch_masks(new_trie: fn() let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); // Compute root to cache initial branch hashes. - let _ = trie.root(); + let _ = trie.root(epoch(0)); // Add a new leaf under the same prefix so non-root branch nodes change. let mut new_key = B256::ZERO; @@ -114,7 +114,7 @@ pub(super) fn test_reveal_nodes_with_branch_masks(new_trie: fn() let mut leaf_updates = SuiteTestHarness::leaf_updates(&changeset); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let _ = trie.root(); + let _ = trie.root(epoch(0)); // Updates should reflect which branch nodes were updated vs removed, // guided by the masks stored during reveal_nodes. @@ -152,7 +152,7 @@ pub(super) fn test_reveal_nodes_skips_on_empty_root(new_trie: fn( // Trie should still be empty. assert_eq!( - trie.root(), + trie.root(epoch(0)), EMPTY_ROOT_HASH, "root should remain EMPTY_ROOT_HASH after reveal on empty root" ); @@ -204,7 +204,7 @@ pub(super) fn test_reveal_nodes_filters_unreachable_boundary_leaves(ne trie.reveal_nodes(&mut proof_nodes).expect("reveal stale proof for key_c should succeed"); // Root must match a reference trie with all 3 keys. - let root = trie.root(); + let root = trie.root(epoch(0)); let expected_storage = BTreeMap::from([(key_a, U256::from(1)), (key_b, insert_value), (key_c, U256::from(3))]); let expected_harness = SuiteTestHarness::new(expected_storage); @@ -322,7 +322,7 @@ pub(super) fn test_remove_then_reveal_does_not_overwrite_collapsed_node( trie.reveal_nodes(&mut stale_proof).expect("revealing stale proof should succeed"); // Root must match a reference trie with all 3 keys. - let root = trie.root(); + let root = trie.root(epoch(0)); let expected_storage = BTreeMap::from([(key_a, U256::from(1)), (key_b, U256::from(2)), (key_c, insert_value)]); let expected_harness = SuiteTestHarness::new(expected_storage); diff --git a/crates/trie/sparse/tests/suite/root.rs b/crates/trie/sparse/tests/suite/root.rs index 64cb94350f6..45f0c89abd3 100644 --- a/crates/trie/sparse/tests/suite/root.rs +++ b/crates/trie/sparse/tests/suite/root.rs @@ -3,7 +3,7 @@ use super::*; /// Calling `root()` on a fresh, empty trie returns `EMPTY_ROOT_HASH`. pub(super) fn test_root_empty_trie(new_trie: fn() -> T) { let mut trie = (new_trie)(); - assert_eq!(trie.root(), EMPTY_ROOT_HASH, "empty trie should return EMPTY_ROOT_HASH"); + assert_eq!(trie.root(epoch(0)), EMPTY_ROOT_HASH, "empty trie should return EMPTY_ROOT_HASH"); } /// Second `root()` call returns cached hash without recomputation. @@ -23,12 +23,12 @@ pub(super) fn test_root_cached_returns_without_recomputation(new_ let harness = SuiteTestHarness::new(storage); let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); - let root1 = trie.root(); + let root1 = trie.root(epoch(0)); assert_eq!(root1, harness.original_root(), "first root should match reference"); assert!(trie.is_root_cached(), "root should be cached after first computation"); - let root2 = trie.root(); + let root2 = trie.root(epoch(0)); assert_eq!(root2, root1, "second root call should return the same cached hash"); } @@ -49,7 +49,7 @@ pub(super) fn test_root_after_single_leaf_update(new_trie: fn() - let harness = SuiteTestHarness::new(storage); let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - let original_root = trie.root(); + let original_root = trie.root(epoch(0)); assert_eq!(original_root, harness.original_root(), "initial root should match reference"); // Modify key_b's value from 2 to 999. @@ -57,7 +57,7 @@ pub(super) fn test_root_after_single_leaf_update(new_trie: fn() - let mut leaf_updates = SuiteTestHarness::leaf_updates(&changes); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let new_root = trie.root(); + let new_root = trie.root(epoch(0)); assert_ne!(new_root, original_root, "root should change after leaf update"); // Build a reference trie with the updated value and verify. @@ -105,7 +105,7 @@ pub(super) fn test_root_deterministic_across_update_orders(new_tr SuiteTestHarness::leaf_updates(&BTreeMap::from([(*key, *value)])); trie.update_leaves(&mut leaf_updates, |_, _| {}).expect("update_leaves should succeed"); } - trie.root() + trie.root(epoch(0)) }; // Build trie B: insert keys in order 5,3,1,4,2. @@ -117,7 +117,7 @@ pub(super) fn test_root_deterministic_across_update_orders(new_tr SuiteTestHarness::leaf_updates(&BTreeMap::from([(*key, *value)])); trie.update_leaves(&mut leaf_updates, |_, _| {}).expect("update_leaves should succeed"); } - trie.root() + trie.root(epoch(0)) }; assert_eq!(root_a, root_b, "root hash should be deterministic regardless of insert order"); @@ -142,9 +142,9 @@ pub(super) fn test_root_handles_small_root_node_without_hash(new_ let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - let root1 = trie.root(); + let root1 = trie.root(epoch(0)); assert_eq!(root1, harness.original_root(), "first root() should match reference trie"); - let root2 = trie.root(); + let root2 = trie.root(epoch(0)); assert_eq!(root2, root1, "second root() should return cached result without panic"); } diff --git a/crates/trie/sparse/tests/suite/set_root.rs b/crates/trie/sparse/tests/suite/set_root.rs index cad0d3a2ced..3d0d57fd54b 100644 --- a/crates/trie/sparse/tests/suite/set_root.rs +++ b/crates/trie/sparse/tests/suite/set_root.rs @@ -15,7 +15,7 @@ pub(super) fn test_set_root_with_branch_node(new_trie: fn() -> T) let harness = SuiteTestHarness::new(storage); let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!(root, harness.original_root()); } @@ -30,7 +30,7 @@ pub(super) fn test_set_root_with_leaf_node(new_trie: fn() -> T) { let root_node = harness.root_node(); let mut trie = (new_trie)(); trie.set_root(root_node.node, root_node.masks, true).expect("set_root should succeed"); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!(root, harness.original_root()); } @@ -50,7 +50,7 @@ pub(super) fn test_set_root_with_extension_node(new_trie: fn() -> let harness = SuiteTestHarness::new(storage); let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!(root, harness.original_root()); } @@ -74,7 +74,7 @@ pub(super) fn test_set_root_retains_updates_when_requested(new_tr let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); // Compute root once so branch hashes are cached. - let _ = trie.root(); + let _ = trie.root(epoch(0)); // Add a new leaf under the same prefix so non-root branch nodes change. let mut new_key = B256::ZERO; @@ -85,7 +85,7 @@ pub(super) fn test_set_root_retains_updates_when_requested(new_tr harness.reveal_and_update(&mut trie, &mut leaf_updates); // Compute root to finalize hashes and generate update actions. - let _ = trie.root(); + let _ = trie.root(epoch(0)); // take_updates should return non-empty updates. let updates = trie.take_updates(); @@ -122,7 +122,7 @@ pub(super) fn test_set_root_does_not_retain_updates_when_not_requested(new_trie: fn() -> T) { let mut trie = (new_trie)(); trie.set_root(TrieNodeV2::EmptyRoot, None, true).expect("set_root should succeed"); - assert_eq!(trie.root(), EMPTY_ROOT_HASH); + assert_eq!(trie.root(epoch(0)), EMPTY_ROOT_HASH); } diff --git a/crates/trie/sparse/tests/suite/size_hint.rs b/crates/trie/sparse/tests/suite/size_hint.rs deleted file mode 100644 index 7bce85455e4..00000000000 --- a/crates/trie/sparse/tests/suite/size_hint.rs +++ /dev/null @@ -1,52 +0,0 @@ -use super::*; - -/// `size_hint` reflects leaf count changes. -/// -/// Builds a 5-leaf trie, records `size_hint`, adds 2 leaves, records again, -/// removes 1 leaf, records again. Asserts s2 > s1 and s3 < s2 (monotonic -/// relative to leaf count changes). -pub(super) fn test_size_hint_reflects_leaf_count(new_trie: fn() -> T) { - let key1 = B256::with_last_byte(0x10); - let key2 = B256::with_last_byte(0x20); - let key3 = B256::with_last_byte(0x30); - let key4 = B256::with_last_byte(0x40); - let key5 = B256::with_last_byte(0x50); - let new_key1 = B256::with_last_byte(0x60); - let new_key2 = B256::with_last_byte(0x70); - - let base_storage: BTreeMap = BTreeMap::from([ - (key1, U256::from(1)), - (key2, U256::from(2)), - (key3, U256::from(3)), - (key4, U256::from(4)), - (key5, U256::from(5)), - ]); - - let harness = SuiteTestHarness::new(base_storage); - - // Include new key targets so proofs cover them. - let all_targets = vec![key1, key2, key3, key4, key5, new_key1, new_key2]; - let mut trie: T = harness.init_trie_with_targets(&all_targets, false, new_trie); - - let s1 = trie.size_hint(); - - // Add 2 new leaves. - let mut leaf_updates = SuiteTestHarness::leaf_updates(&BTreeMap::from([ - (new_key1, U256::from(6)), - (new_key2, U256::from(7)), - ])); - harness.reveal_and_update(&mut trie, &mut leaf_updates); - assert!(leaf_updates.is_empty(), "leaf_updates should be drained after insert"); - - let s2 = trie.size_hint(); - assert!(s2 > s1, "size_hint should increase after adding leaves: s2={s2} > s1={s1}"); - - // Remove 1 leaf (set value to zero). - let mut remove_updates = - SuiteTestHarness::leaf_updates(&BTreeMap::from([(new_key2, U256::ZERO)])); - harness.reveal_and_update(&mut trie, &mut remove_updates); - assert!(remove_updates.is_empty(), "leaf_updates should be drained after removal"); - - let s3 = trie.size_hint(); - assert!(s3 < s2, "size_hint should decrease after removing a leaf: s3={s3} < s2={s2}"); -} diff --git a/crates/trie/sparse/tests/suite/take_updates.rs b/crates/trie/sparse/tests/suite/take_updates.rs index ed56a6c8199..581d7097b5e 100644 --- a/crates/trie/sparse/tests/suite/take_updates.rs +++ b/crates/trie/sparse/tests/suite/take_updates.rs @@ -35,7 +35,7 @@ pub(super) fn test_take_updates_resets_after_take(new_trie: fn() let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); // Cache initial branch hashes. - let _ = trie.root(); + let _ = trie.root(epoch(0)); // Round 1: add a new leaf A under the shared prefix, root, take. let mut key_a = B256::ZERO; @@ -44,7 +44,7 @@ pub(super) fn test_take_updates_resets_after_take(new_trie: fn() let changeset_a: BTreeMap = BTreeMap::from([(key_a, U256::from(999))]); let mut leaf_updates_a = SuiteTestHarness::leaf_updates(&changeset_a); harness.reveal_and_update(&mut trie, &mut leaf_updates_a); - let _ = trie.root(); + let _ = trie.root(epoch(0)); let updates1 = trie.take_updates(); assert!( @@ -67,7 +67,7 @@ pub(super) fn test_take_updates_resets_after_take(new_trie: fn() let changeset_b: BTreeMap = BTreeMap::from([(key_b, U256::from(888))]); let mut leaf_updates_b = SuiteTestHarness::leaf_updates(&changeset_b); harness.reveal_and_update(&mut trie, &mut leaf_updates_b); - let _ = trie.root(); + let _ = trie.root(epoch(0)); let updates2 = trie.take_updates(); assert!( @@ -129,7 +129,7 @@ pub(super) fn test_take_updates_contains_updated_and_removed_nodes = [(key_new_a, val), (key_new_b, val)].into_iter().collect(); let mut leaf_updates = SuiteTestHarness::leaf_updates(&changeset1); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let _ = trie.root(); + let _ = trie.root(epoch(0)); // Do NOT call take_updates() — updates accumulate across root() calls. @@ -251,7 +251,7 @@ pub(super) fn test_take_updates_cross_cancellation_across_root_calls(new_trie: fn() - // The update map should be drained on success. assert!(leaf_updates.is_empty(), "leaf_updates should be drained after successful insert"); - let root = trie.root(); + let root = trie.root(epoch(0)); // Compute expected root with all 4 leaves. let expected_storage = BTreeMap::from([ @@ -66,7 +66,7 @@ pub(super) fn test_update_leaves_modify_existing_leaf(new_trie: f assert!(leaf_updates.is_empty(), "leaf_updates should be drained after successful modify"); - let root = trie.root(); + let root = trie.root(epoch(0)); // Compute expected root with the modified value. let expected_storage = @@ -98,7 +98,7 @@ pub(super) fn test_insert_single_leaf_into_empty_trie(new_trie: f assert!(leaf_updates.is_empty(), "leaf_updates should be drained"); - let root = trie.root(); + let root = trie.root(epoch(0)); let expected_harness = SuiteTestHarness::new(BTreeMap::from([(key, value)])); assert_eq!( @@ -136,7 +136,7 @@ pub(super) fn test_insert_multiple_leaves_into_empty_trie(new_tri assert!(leaf_updates.is_empty(), "leaf_updates should be drained"); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!(root, expected_harness.original_root(), "root should match reference 256-leaf trie"); let updates = trie.take_updates(); @@ -174,7 +174,7 @@ pub(super) fn test_update_all_leaves_with_new_values(new_trie: fn }) .expect("update_leaves should succeed"); - let hash1 = trie.root(); + let hash1 = trie.root(epoch(0)); assert_eq!(hash1, expected_old.original_root(), "hash1 should match reference with old values"); // Update all 256 keys with new values. @@ -184,7 +184,7 @@ pub(super) fn test_update_all_leaves_with_new_values(new_trie: fn }) .expect("update_leaves should succeed"); - let hash2 = trie.root(); + let hash2 = trie.root(epoch(0)); assert_eq!(hash2, expected_new.original_root(), "hash2 should match reference with new values"); assert_ne!(hash1, hash2, "roots should differ after updating all values"); } @@ -212,7 +212,7 @@ pub(super) fn test_two_leaves_at_adjacent_keys_root_correctness( panic!("no proof callback expected on empty trie"); }) .expect("update_leaves should succeed"); - trie.root(); + trie.root(epoch(0)); // Insert second leaf and compute root. let mut leaf_updates = SuiteTestHarness::leaf_updates(&BTreeMap::from([(key_51, value)])); @@ -220,7 +220,7 @@ pub(super) fn test_two_leaves_at_adjacent_keys_root_correctness( panic!("no proof callback expected — all paths already revealed"); }) .expect("update_leaves should succeed"); - let root = trie.root(); + let root = trie.root(epoch(0)); let expected = SuiteTestHarness::new(BTreeMap::from([(key_50, value), (key_51, value)])); assert_eq!(root, expected.original_root(), "root should match reference two-leaf trie"); @@ -253,7 +253,7 @@ pub(super) fn test_update_leaves_remove_leaf(new_trie: fn() -> T) assert!(leaf_updates.is_empty(), "leaf_updates should be drained after successful removal"); - let root = trie.root(); + let root = trie.root(epoch(0)); // Expected: reference trie with only key1 and key3. let expected_storage = BTreeMap::from([(key1, U256::from(1)), (key3, U256::from(3))]); @@ -299,7 +299,7 @@ pub(super) fn test_remove_leaf_branch_collapses_to_extension(new_ let mut leaf_updates = SuiteTestHarness::leaf_updates(&BTreeMap::from([(key_537, U256::ZERO)])); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let root = trie.root(); + let root = trie.root(epoch(0)); // Expected: reference trie with only the two remaining leaves. let expected_storage = BTreeMap::from([(key_50231, U256::from(1)), (key_50233, U256::from(2))]); @@ -326,14 +326,14 @@ pub(super) fn test_remove_leaf_branch_collapses_to_leaf(new_trie: let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); // Compute root to cache hashes and drain initial updates. - let _ = trie.root(); + let _ = trie.root(epoch(0)); let _ = trie.take_updates(); // Remove key_a → branch collapses to a single leaf (key_b). let mut leaf_updates = SuiteTestHarness::leaf_updates(&BTreeMap::from([(key_a, U256::ZERO)])); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let root = trie.root(); + let root = trie.root(epoch(0)); // Expected: reference trie with only key_b. let expected_storage = BTreeMap::from([(key_b, U256::from(200))]); @@ -369,7 +369,7 @@ pub(super) fn test_remove_last_leaf_produces_empty_root(new_trie: let mut leaf_updates = SuiteTestHarness::leaf_updates(&BTreeMap::from([(key, U256::ZERO)])); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!(root, EMPTY_ROOT_HASH, "removing the only leaf should produce EMPTY_ROOT_HASH"); } @@ -407,7 +407,7 @@ pub(super) fn test_insert_then_remove_sequence(new_trie: fn() -> let mut leaf_updates = SuiteTestHarness::leaf_updates(&base_storage); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let root_after_insert = trie.root(); + let root_after_insert = trie.root(epoch(0)); assert_eq!(root_after_insert, harness.original_root(), "root after inserting all 6 leaves"); // Remove leaves one at a time in the same order as the original test: k3, k1, k4, k6, k2, k5. @@ -418,7 +418,7 @@ pub(super) fn test_insert_then_remove_sequence(new_trie: fn() -> let mut leaf_updates = SuiteTestHarness::leaf_updates(&changeset); harness.reveal_and_update(&mut trie, &mut leaf_updates); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!( root, harness.original_root(), @@ -429,7 +429,7 @@ pub(super) fn test_insert_then_remove_sequence(new_trie: fn() -> } // After all removals, trie should be empty. - assert_eq!(trie.root(), EMPTY_ROOT_HASH, "final root should be EMPTY_ROOT_HASH"); + assert_eq!(trie.root(epoch(0)), EMPTY_ROOT_HASH, "final root should be EMPTY_ROOT_HASH"); } /// Removing a nonexistent key should not invalidate cached hashes. @@ -449,7 +449,7 @@ pub(super) fn test_remove_nonexistent_leaf_preserves_hashes(new_t let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); // Compute root to cache hashes on all nodes. - let root_before = trie.root(); + let root_before = trie.root(epoch(0)); assert_eq!(root_before, harness.original_root()); // Try to remove a key that doesn't exist in the trie. @@ -459,7 +459,7 @@ pub(super) fn test_remove_nonexistent_leaf_preserves_hashes(new_t harness.reveal_and_update(&mut trie, &mut leaf_updates); // Root should be identical — cached hashes should be preserved. - let root_after = trie.root(); + let root_after = trie.root(epoch(0)); assert_eq!( root_before, root_after, "removing a nonexistent leaf should preserve cached hashes and return the same root" @@ -578,7 +578,7 @@ pub(super) fn test_update_leaves_retry_after_reveal(new_trie: fn( expected_storage.insert(target_key, new_value); let expected_harness = SuiteTestHarness::new(expected_storage); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!( root, expected_harness.original_root(), @@ -637,7 +637,7 @@ pub(super) fn test_remove_leaf_blinded_sibling_requires_reveal(ne expected_storage.remove(&revealed_key); let expected_harness = SuiteTestHarness::new(expected_storage); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!( root, expected_harness.original_root(), @@ -645,11 +645,9 @@ pub(super) fn test_remove_leaf_blinded_sibling_requires_reveal(ne ); } -/// Atomic rollback — node structure and `prefix_set` unchanged after -/// removal of a revealed leaf when the sibling is blinded. +/// Atomic rollback preserves a revealed leaf when its sibling is blinded. /// -/// Same scenario as the value-preservation test, but additionally checks that -/// `size_hint` (node count) is unchanged and the update remains in the map. +/// The update remains pending until the sibling can be revealed. pub(super) fn test_update_leaves_removal_branch_collapse_blinded_sibling( new_trie: fn() -> T, ) { @@ -678,7 +676,6 @@ pub(super) fn test_update_leaves_removal_branch_collapse_blinded_sibling(new_trie: let harness = SuiteTestHarness::new(base_storage); let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); - let root_before = trie.root(); + let root_before = trie.root(epoch(0)); // Call update_leaves with Touched for an existing key. let mut leaf_updates: B256Map = once((key2, LeafUpdate::Touched)).collect(); @@ -849,7 +843,7 @@ pub(super) fn test_update_leaves_touched_fully_revealed(new_trie: // Key should be drained from the map. assert!(leaf_updates.is_empty(), "Touched key should be drained from updates map"); // Root should be unchanged. - assert_eq!(trie.root(), root_before, "root should be unchanged after Touched no-op"); + assert_eq!(trie.root(epoch(0)), root_before, "root should be unchanged after Touched no-op"); } /// `LeafUpdate::Touched` on a path with a blinded node should @@ -880,7 +874,7 @@ pub(super) fn test_update_leaves_touched_blinded_requests_proof( // Reveal only group_a keys, leaving group_b's subtrie blinded. let mut trie: T = harness.init_trie_with_targets(&group_a_keys, false, new_trie); - let root_before = trie.root(); + let root_before = trie.root(epoch(0)); // Submit Touched for a key in group_b's blinded subtrie. let target_key = { @@ -901,7 +895,11 @@ pub(super) fn test_update_leaves_touched_blinded_requests_proof( // Key should remain in the updates map. assert!(!leaf_updates.is_empty(), "Touched key should remain in map when blinded"); // Root should be unchanged (no mutation). - assert_eq!(trie.root(), root_before, "root should be unchanged after Touched on blinded path"); + assert_eq!( + trie.root(epoch(0)), + root_before, + "root should be unchanged after Touched on blinded path" + ); } /// Touched on a nonexistent key in an empty trie is drained silently. @@ -945,7 +943,7 @@ pub(super) fn test_update_leaves_touched_nonexistent_in_populated_trie(new_trie: // apply_changeset merged into existing storage — need a fresh harness for the expected state. let expected_harness = SuiteTestHarness::new(expected_storage); - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!( root, expected_harness.original_root(), @@ -1075,7 +1073,7 @@ pub(super) fn test_remove_leaf_marks_ancestors_dirty_unconditionally(new_trie: fn() -> T) { // Create a trie with 10 leaves across different first-nibble subtries. - // We'll prune some subtries, then remove a leaf whose branch collapse - // involves a sibling in a pruned (blinded) subtrie. let mut storage: BTreeMap = BTreeMap::new(); let mut keys = Vec::new(); for i in 0u8..10 { @@ -1267,22 +1263,13 @@ pub(super) fn test_remove_leaf_does_not_reveal_blind_subtries(new } let mut harness = SuiteTestHarness::new(storage.clone()); - let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); + let mut trie: T = harness.init_trie_with_targets(&keys[..2], true, new_trie); // Compute initial root and drain initial updates. - let _ = trie.root(); + let _ = trie.root(epoch(0)); let _ = trie.take_updates(); - // Prune: retain only keys[0] and keys[1] (nibbles 0x0 and 0x1). - // All other subtries become blinded. - let retained: Vec = [keys[0], keys[1]].iter().map(|k| Nibbles::unpack(*k)).collect(); - trie.prune(&retained); - - // Verify root is unchanged after prune. - let root_after_prune = trie.root(); - assert_eq!(root_after_prune, harness.original_root(), "root should be unchanged after prune"); - - // Now remove keys[0] — this leaves keys[1] and all the blinded subtries. + // Remove keys[0] — this leaves keys[1] and all the blinded subtries. // The branch at the root still has multiple children (keys[1] + blinded children), // so this should not trigger a problematic branch collapse into blinded territory. let removal: BTreeMap = once((keys[0], U256::ZERO)).collect(); @@ -1290,7 +1277,7 @@ pub(super) fn test_remove_leaf_does_not_reveal_blind_subtries(new let mut removal_updates = SuiteTestHarness::leaf_updates(&removal); harness.reveal_and_update(&mut trie, &mut removal_updates); - let root_after_removal = trie.root(); + let root_after_removal = trie.root(epoch(0)); assert_eq!( root_after_removal, harness.original_root(), @@ -1302,22 +1289,22 @@ pub(super) fn test_remove_leaf_does_not_reveal_blind_subtries(new let find_result = trie.find_leaf(&removed_nibbles, None).expect("find_leaf should succeed"); assert_eq!(find_result, LeafLookup::NonExistent, "removed leaf should not be found"); - // Verify that the retained leaf (keys[1]) is still accessible. - let retained_nibbles = Nibbles::unpack(keys[1]); - let find_result = trie.find_leaf(&retained_nibbles, None).expect("find_leaf should succeed"); - assert_eq!(find_result, LeafLookup::Exists, "retained leaf should still be findable"); + // Verify that the other revealed leaf (keys[1]) is still accessible. + let remaining_nibbles = Nibbles::unpack(keys[1]); + let find_result = trie.find_leaf(&remaining_nibbles, None).expect("find_leaf should succeed"); + assert_eq!(find_result, LeafLookup::Exists, "remaining leaf should still be findable"); - // Now update the retained leaf to verify the trie is in a consistent state. + // Now update the remaining leaf to verify the trie is in a consistent state. let modification: BTreeMap = once((keys[1], U256::from(999))).collect(); harness.apply_changeset(modification.clone()); let mut mod_updates = SuiteTestHarness::leaf_updates(&modification); harness.reveal_and_update(&mut trie, &mut mod_updates); - let root_after_mod = trie.root(); + let root_after_mod = trie.root(epoch(0)); assert_eq!( root_after_mod, harness.original_root(), - "root after modifying retained leaf should match reference" + "root after modifying remaining leaf should match reference" ); } @@ -1396,7 +1383,7 @@ pub(super) fn test_branch_collapse_multi_empty_subtries_blinded_remaining( let mut trie: T = harness.init_trie_with_targets(&revealed_keys, false, new_trie); // Verify initial root matches. - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!(root, harness.original_root(), "initial root mismatch"); // Delete both 0xAB leaves + Touched on a third 0xAB key (not in the trie). @@ -1473,7 +1460,7 @@ pub(super) fn test_subtrie_collapse_touched_with_blinded_sibling( expected_storage.remove(&key_ab2); let expected_harness = SuiteTestHarness::new(expected_storage); - let actual_root = trie.root(); + let actual_root = trie.root(epoch(0)); assert_eq!(actual_root, expected_harness.original_root(), "post-delete root mismatch"); } @@ -1514,7 +1501,7 @@ pub(super) fn test_subtrie_emptied_by_deletes_with_touched(new_tr let mut trie: T = harness.init_trie_with_targets(&all_keys, false, new_trie); // Verify initial root matches. - let root = trie.root(); + let root = trie.root(epoch(0)); assert_eq!(root, harness.original_root(), "initial root mismatch"); // Delete both 0xAB leaves + Touched on a third 0xAB key (not in the trie). @@ -1535,6 +1522,6 @@ pub(super) fn test_subtrie_emptied_by_deletes_with_touched(new_tr expected_storage.remove(&key_ab2); let expected_harness = SuiteTestHarness::new(expected_storage); - let actual_root = trie.root(); + let actual_root = trie.root(epoch(0)); assert_eq!(actual_root, expected_harness.original_root(), "post-delete root mismatch"); } diff --git a/crates/trie/sparse/tests/suite/wipe_clear.rs b/crates/trie/sparse/tests/suite/wipe_clear.rs index 5a4ff2d4adb..f3c789a93cd 100644 --- a/crates/trie/sparse/tests/suite/wipe_clear.rs +++ b/crates/trie/sparse/tests/suite/wipe_clear.rs @@ -15,13 +15,13 @@ pub(super) fn test_wipe_resets_to_empty_root(new_trie: fn() -> T) let mut trie: T = harness.init_trie_fully_revealed(false, new_trie); // Compute root to confirm the trie is populated. - let root_before = trie.root(); + let root_before = trie.root(epoch(0)); assert_eq!(root_before, harness.original_root()); assert_ne!(root_before, EMPTY_ROOT_HASH); // Wipe and verify empty root. trie.wipe(); - let root_after = trie.root(); + let root_after = trie.root(epoch(0)); assert_eq!(root_after, EMPTY_ROOT_HASH, "root must be EMPTY_ROOT_HASH after wipe"); } @@ -42,13 +42,13 @@ pub(super) fn test_clear_resets_trie_but_preserves_update_tracking(new_trie: fn() -> let mut trie: T = harness.init_trie_fully_revealed(true, new_trie); // Compute root to populate the trie fully. - let root_before = trie.root(); + let root_before = trie.root(epoch(0)); assert_eq!(root_before, harness.original_root()); assert_ne!(root_before, EMPTY_ROOT_HASH); @@ -81,7 +81,7 @@ pub(super) fn test_wipe_produces_wiped_updates(new_trie: fn() -> trie.wipe(); // Root should be EMPTY_ROOT_HASH after wipe. - let root_after = trie.root(); + let root_after = trie.root(epoch(0)); assert_eq!(root_after, EMPTY_ROOT_HASH, "root must be EMPTY_ROOT_HASH after wipe"); // take_updates should return updates with the wiped flag set. @@ -104,13 +104,13 @@ pub(super) fn test_clear_then_reuse_trie(new_trie: fn() -> T) { let harness_1 = SuiteTestHarness::new(storage_1); let mut trie: T = harness_1.init_trie_fully_revealed(false, new_trie); - let root_1 = trie.root(); + let root_1 = trie.root(epoch(0)); assert_eq!(root_1, harness_1.original_root()); assert_ne!(root_1, EMPTY_ROOT_HASH); // Phase 2: clear the trie and verify empty. trie.clear(); - assert_eq!(trie.root(), EMPTY_ROOT_HASH, "root must be EMPTY_ROOT_HASH after clear"); + assert_eq!(trie.root(epoch(0)), EMPTY_ROOT_HASH, "root must be EMPTY_ROOT_HASH after clear"); // Phase 3: build a completely different dataset with 3 leaves. let storage_2: BTreeMap = BTreeMap::from([ @@ -138,7 +138,7 @@ pub(super) fn test_clear_then_reuse_trie(new_trie: fn() -> T) { harness_2.reveal_and_update(&mut trie, &mut leaf_updates); // Compute root and verify against reference. - let root_2 = trie.root(); + let root_2 = trie.root(epoch(0)); // Update the reference harness with the 4th leaf. harness_2.apply_changeset(changeset); diff --git a/crates/trie/trie/src/witness.rs b/crates/trie/trie/src/witness.rs index 226e5a22d87..41ea9a067cf 100644 --- a/crates/trie/trie/src/witness.rs +++ b/crates/trie/trie/src/witness.rs @@ -18,7 +18,7 @@ use reth_trie_common::{ DecodedMultiProofV2, ExecutionWitnessMode, HashedPostState, MultiProofTargetsV2, ProofV2Target, TrieNodeV2, }; -use reth_trie_sparse::{LeafUpdate, SparseStateTrie, SparseTrie as _}; +use reth_trie_sparse::{LeafUpdate, SparseStateTrie, SparseTrie as _, TrieNodeEpoch}; /// State transition witness for the trie. #[derive(Debug)] @@ -251,7 +251,7 @@ where let storage_root = if let Some(storage_trie) = sparse_trie.storage_trie_mut(&hashed_address) { - storage_trie.root() + storage_trie.root(TrieNodeEpoch::UNMODIFIED) } else { let record_root_node = !self.mode.is_canonical() || state From 75e4e629d9c2b1cd2c11959a5ef53b852bb7e5d4 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Mon, 27 Jul 2026 17:49:38 +0200 Subject: [PATCH 051/148] refactor(storage): extract reth-storage-overlay crate (#26506) --- .github/CODEOWNERS | 1 + .github/scripts/check_wasm.sh | 1 + Cargo.lock | 42 +- Cargo.toml | 2 + crates/chain-state/Cargo.toml | 6 +- crates/chain-state/src/lib.rs | 3 - crates/engine/tree/Cargo.toml | 5 +- crates/engine/tree/src/launch.rs | 5 +- crates/engine/tree/src/tree/mod.rs | 10 +- .../engine/tree/src/tree/payload_validator.rs | 21 +- crates/engine/tree/src/tree/state.rs | 34 +- .../tree/src/tree/state_root_strategy/mod.rs | 23 +- .../tree/state_root_strategy/sparse_trie.rs | 5 +- crates/engine/tree/src/tree/tests.rs | 11 +- crates/node/builder/Cargo.toml | 3 +- crates/node/builder/src/launch/common.rs | 2 +- crates/node/builder/src/launch/engine.rs | 5 +- crates/node/builder/src/rpc.rs | 8 +- crates/storage/provider/Cargo.toml | 6 +- .../src/providers/blockchain_provider.rs | 5 +- .../provider/src/providers/database/mod.rs | 2 +- .../src/providers/database/provider.rs | 3 +- crates/storage/provider/src/providers/mod.rs | 2 +- .../src/providers/state/historical.rs | 8 +- .../provider/src/providers/state/overlay.rs | 799 +---------- crates/storage/storage-overlay/Cargo.toml | 53 + crates/storage/storage-overlay/src/builder.rs | 835 +++++++++++ .../storage-overlay/src/changeset_cache.rs | 1160 ++++++++++++++++ crates/storage/storage-overlay/src/lib.rs | 12 + .../storage-overlay/src/manager.rs} | 64 +- crates/trie/db/Cargo.toml | 10 +- crates/trie/db/src/changesets.rs | 1229 +---------------- crates/trie/parallel/Cargo.toml | 3 +- crates/trie/parallel/src/proof_task.rs | 4 +- examples/custom-state-root/Cargo.toml | 2 +- examples/custom-state-root/src/main.rs | 8 +- 36 files changed, 2277 insertions(+), 2115 deletions(-) create mode 100644 crates/storage/storage-overlay/Cargo.toml create mode 100644 crates/storage/storage-overlay/src/builder.rs create mode 100644 crates/storage/storage-overlay/src/changeset_cache.rs create mode 100644 crates/storage/storage-overlay/src/lib.rs rename crates/{chain-state/src/state_trie_overlay.rs => storage/storage-overlay/src/manager.rs} (93%) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 65d6fc7e40d..6887f88baf4 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -38,6 +38,7 @@ crates/storage/libmdbx-rs/ @shekhirin crates/storage/nippy-jar/ @joshieDo @shekhirin crates/storage/provider/ @joshieDo @shekhirin crates/storage/storage-api/ @joshieDo +crates/storage/storage-overlay/ @joshieDo @shekhirin @mediocregopher crates/tasks/ @mattsse @DaniPopes crates/tokio-util/ @mattsse crates/tracing/ @mattsse @shekhirin diff --git a/.github/scripts/check_wasm.sh b/.github/scripts/check_wasm.sh index 8c46933325f..13bea47cfea 100755 --- a/.github/scripts/check_wasm.sh +++ b/.github/scripts/check_wasm.sh @@ -84,6 +84,7 @@ exclude_crates=( reth-nippy-jar # zstd-sys (direct dependency) reth-node-types # zstd-sys via reth-codecs -> reth-zstd-compressors reth-rpc-server-types # secp256k1-sys via reth-network-api -> enr + reth-storage-overlay # zstd-sys via reth-trie-db reth-trie-db # zstd-sys via reth-codecs -> reth-zstd-compressors ) diff --git a/Cargo.lock b/Cargo.lock index f75716d4972..f7d7054af59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3629,12 +3629,12 @@ dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", "eyre", - "reth-chain-state", "reth-engine-tree", "reth-ethereum", "reth-evm", "reth-primitives-traits", "reth-provider", + "reth-storage-overlay", "reth-trie", "serde_json", "tokio", @@ -7688,7 +7688,6 @@ dependencies = [ "parking_lot", "pin-project", "rand 0.9.4", - "rayon", "reth-chainspec", "reth-errors", "reth-ethereum-primitives", @@ -7696,7 +7695,6 @@ dependencies = [ "reth-metrics", "reth-primitives-traits", "reth-storage-api", - "reth-tasks", "reth-testing-utils", "reth-trie", "reth-trie-sparse", @@ -8373,12 +8371,12 @@ dependencies = [ "reth-stages", "reth-stages-api", "reth-static-file", + "reth-storage-overlay", "reth-tasks", "reth-testing-utils", "reth-tracing", "reth-trie", "reth-trie-common", - "reth-trie-db", "reth-trie-parallel", "reth-trie-sparse", "revm", @@ -9293,11 +9291,11 @@ dependencies = [ "reth-rpc-layer", "reth-stages", "reth-static-file", + "reth-storage-overlay", "reth-tasks", "reth-tokio-util", "reth-tracing", "reth-transaction-pool", - "reth-trie-db", "secp256k1 0.30.0", "serde_json", "tempfile", @@ -9663,6 +9661,7 @@ dependencies = [ "reth-static-file-types", "reth-storage-api", "reth-storage-errors", + "reth-storage-overlay", "reth-tasks", "reth-testing-utils", "reth-tokio-util", @@ -10331,6 +10330,34 @@ dependencies = [ "thiserror 2.0.18", ] +[[package]] +name = "reth-storage-overlay" +version = "2.4.1" +dependencies = [ + "alloy-consensus", + "alloy-eips", + "alloy-primitives", + "metrics", + "parking_lot", + "rayon", + "reth-chain-state", + "reth-db", + "reth-errors", + "reth-ethereum-primitives", + "reth-metrics", + "reth-primitives-traits", + "reth-provider", + "reth-prune-types", + "reth-stages-types", + "reth-storage-api", + "reth-storage-errors", + "reth-tasks", + "reth-trie", + "reth-trie-common", + "reth-trie-db", + "tracing", +] + [[package]] name = "reth-storage-rpc-provider" version = "2.4.1" @@ -10563,15 +10590,12 @@ dependencies = [ "alloy-consensus", "alloy-primitives", "alloy-rlp", - "metrics", - "parking_lot", "proptest", "proptest-arbitrary-interop", "reth-chainspec", "reth-db", "reth-db-api", "reth-execution-errors", - "reth-metrics", "reth-primitives-traits", "reth-provider", "reth-stages-types", @@ -10604,9 +10628,9 @@ dependencies = [ "reth-primitives-traits", "reth-provider", "reth-storage-errors", + "reth-storage-overlay", "reth-tasks", "reth-trie", - "reth-trie-db", "reth-trie-sparse", "revm", "thiserror 2.0.18", diff --git a/Cargo.toml b/Cargo.toml index 42c5f878b47..beb4be10fc2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -106,6 +106,7 @@ members = [ "crates/storage/nippy-jar/", "crates/storage/provider/", "crates/storage/storage-api/", + "crates/storage/storage-overlay/", "crates/tasks/", "crates/tokio-util/", "crates/tracing/", @@ -413,6 +414,7 @@ reth-static-file = { path = "crates/static-file/static-file" } reth-static-file-types = { path = "crates/static-file/types", default-features = false } reth-storage-api = { path = "crates/storage/storage-api", default-features = false } reth-storage-errors = { path = "crates/storage/errors", default-features = false } +reth-storage-overlay = { path = "crates/storage/storage-overlay" } reth-tasks = { path = "crates/tasks" } reth-testing-utils = { path = "testing/testing-utils" } reth-tokio-util = { path = "crates/tokio-util" } diff --git a/crates/chain-state/Cargo.toml b/crates/chain-state/Cargo.toml index 3f218fa9265..4a4b494821f 100644 --- a/crates/chain-state/Cargo.toml +++ b/crates/chain-state/Cargo.toml @@ -18,9 +18,8 @@ reth-errors.workspace = true reth-execution-types.workspace = true reth-metrics.workspace = true reth-ethereum-primitives.workspace = true -reth-primitives-traits = { workspace = true, features = ["dashmap"] } +reth-primitives-traits.workspace = true reth-storage-api.workspace = true -reth-tasks = { workspace = true, features = ["rayon"], optional = true } reth-trie-sparse = { workspace = true, features = ["std"] } reth-trie.workspace = true @@ -42,7 +41,6 @@ derive_more.workspace = true metrics.workspace = true parking_lot.workspace = true pin-project.workspace = true -rayon = { workspace = true, optional = true } serde = { workspace = true, optional = true } # optional deps for test-utils @@ -81,7 +79,5 @@ test-utils = [ "reth-primitives-traits/test-utils", "reth-trie/test-utils", "reth-ethereum-primitives/test-utils", - "reth-tasks?/test-utils", "reth-trie-sparse/test-utils", ] -rayon = ["dep:rayon", "dep:reth-tasks"] diff --git a/crates/chain-state/src/lib.rs b/crates/chain-state/src/lib.rs index 5844090fd7c..c13232504a4 100644 --- a/crates/chain-state/src/lib.rs +++ b/crates/chain-state/src/lib.rs @@ -14,9 +14,6 @@ pub use execution_stats::ExecutionTimingStats; mod in_memory; pub use in_memory::*; -mod state_trie_overlay; -pub use state_trie_overlay::*; - mod preserved_sparse_trie; pub use preserved_sparse_trie::*; diff --git a/crates/engine/tree/Cargo.toml b/crates/engine/tree/Cargo.toml index 95cb5dbc21d..f3e605bb2e3 100644 --- a/crates/engine/tree/Cargo.toml +++ b/crates/engine/tree/Cargo.toml @@ -12,7 +12,7 @@ workspace = true [dependencies] # reth -reth-chain-state = { workspace = true, features = ["rayon"] } +reth-chain-state.workspace = true reth-chainspec = { workspace = true, optional = true } reth-consensus.workspace = true reth-db.workspace = true @@ -30,12 +30,12 @@ reth-provider.workspace = true reth-prune.workspace = true reth-revm = { workspace = true, features = ["optional-balance-check", "optional-no-base-fee"] } reth-stages-api.workspace = true +reth-storage-overlay = { workspace = true, features = ["rayon"] } reth-tasks = { workspace = true, features = ["rayon"] } reth-trie-parallel.workspace = true reth-trie-sparse = { workspace = true, features = ["std", "metrics"] } reth-trie.workspace = true reth-trie-common.workspace = true -reth-trie-db.workspace = true # alloy alloy-evm.workspace = true @@ -122,7 +122,6 @@ test-utils = [ "reth-tracing", "reth-trie/test-utils", "reth-trie-common/test-utils", - "reth-trie-db/test-utils", "reth-trie-sparse/test-utils", "reth-prune-types?/test-utils", "reth-trie-parallel/test-utils", diff --git a/crates/engine/tree/src/launch.rs b/crates/engine/tree/src/launch.rs index 826bb307051..d2a6e6c9fe0 100644 --- a/crates/engine/tree/src/launch.rs +++ b/crates/engine/tree/src/launch.rs @@ -13,7 +13,6 @@ use crate::{ tree::{EngineApiTreeHandler, EngineValidator, TreeConfig, WaitForCaches}, }; use futures::Stream; -use reth_chain_state::StateTrieOverlayManager; use reth_consensus::FullConsensus; use reth_engine_primitives::BeaconEngineMessage; use reth_evm::ConfigureEvm; @@ -26,8 +25,8 @@ use reth_provider::{ }; use reth_prune::PrunerWithFactory; use reth_stages_api::{MetricEventsSender, Pipeline}; +use reth_storage_overlay::{ChangesetCache, OverlayManager}; use reth_tasks::Runtime; -use reth_trie_db::ChangesetCache; use std::sync::Arc; /// Builds the engine [`ChainOrchestrator`] that drives the chain forward. @@ -61,7 +60,7 @@ pub fn build_engine_orchestrator( pruner: PrunerWithFactory>, payload_builder: PayloadBuilderHandle, payload_validator: V, - state_trie_overlays: StateTrieOverlayManager, + state_trie_overlays: OverlayManager, tree_config: TreeConfig, sync_metrics_tx: MetricEventsSender, evm_config: C, diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index f5b0b840365..434226a28fb 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -14,7 +14,7 @@ use alloy_rpc_types_engine::{ use error::{InsertBlockError, InsertBlockFatalError, InsertBlockValidationError}; use reth_chain_state::{ CanonicalInMemoryState, ExecutedBlock, ExecutionTimingStats, MemoryOverlayStateProvider, - NewCanonicalChain, StateTrieOverlayManager, + NewCanonicalChain, }; use reth_consensus::{Consensus, FullConsensus}; use reth_engine_primitives::{ @@ -36,9 +36,9 @@ use reth_provider::{ }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; +use reth_storage_overlay::{ChangesetCache, OverlayManager}; use reth_tasks::{spawn_os_thread, utils::increase_thread_priority}; use reth_trie::ComputedTrieData; -use reth_trie_db::ChangesetCache; use revm::interpreter::debug_unreachable; use state::TreeState; use std::{fmt::Debug, ops, sync::Arc, time::Duration}; @@ -165,7 +165,7 @@ impl EngineApiTreeState { invalid_header_hit_eviction_threshold: u8, canonical_block: BlockNumHash, engine_kind: EngineApiKind, - state_trie_overlays: StateTrieOverlayManager, + state_trie_overlays: OverlayManager, ) -> Self { Self { invalid_headers: InvalidHeaderCache::new( @@ -468,7 +468,7 @@ where persistence: PersistenceHandle, payload_builder: PayloadBuilderHandle, canonical_in_memory_state: CanonicalInMemoryState, - state_trie_overlays: StateTrieOverlayManager, + state_trie_overlays: OverlayManager, config: TreeConfig, kind: EngineApiKind, evm_config: C, @@ -2238,7 +2238,7 @@ where "computing block trie updates", ); let db_provider = self.provider.database_provider_ro()?; - let trie_updates = reth_trie_db::compute_block_trie_updates( + let trie_updates = reth_storage_overlay::compute_block_trie_updates( &self.changeset_cache, &db_provider, block.number(), diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index a31aa7f6f00..6b616481e21 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -125,9 +125,7 @@ use crate::tree::{ }; use alloy_consensus::constants::KECCAK_EMPTY; use alloy_primitives::Address; -use reth_chain_state::{ - CanonicalInMemoryState, ExecutedBlock, ExecutionTimingStats, StateTrieOverlayManager, -}; +use reth_chain_state::{CanonicalInMemoryState, ExecutedBlock, ExecutionTimingStats}; use reth_consensus::{ConsensusError, FullConsensus, ReceiptRootBloom}; use reth_engine_primitives::{ ConfigureEngineEvm, ExecutableTxIterator, ExecutionPayload, InvalidBlockHook, PayloadValidator, @@ -148,18 +146,17 @@ use reth_primitives_traits::{ RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, }; use reth_provider::{ - providers::{OverlayBuilder, OverlayStateProviderFactory}, - BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, - DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader, - StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader, - StorageChangeSetReader, StorageSettingsCache, + providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader, + ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, + ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderBox, + StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; +use reth_storage_overlay::{ChangesetCache, OverlayBuilder, OverlayManager}; use reth_trie::{ hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, LazyTrieData, }; -use reth_trie_db::ChangesetCache; use std::{ sync::{ atomic::{AtomicUsize, Ordering}, @@ -292,7 +289,7 @@ where /// Task runtime for spawning parallel work. runtime: reth_tasks::Runtime, /// Shared state trie in-memory overlay data. - state_trie_overlays: StateTrieOverlayManager, + state_trie_overlays: OverlayManager, /// State-root strategy used to prepare per-block commitment tasks. #[debug(skip)] state_root_strategy: Arc>, @@ -341,7 +338,7 @@ where config: TreeConfig, invalid_block_hook: Box>, changeset_cache: ChangesetCache, - state_trie_overlays: StateTrieOverlayManager, + state_trie_overlays: OverlayManager, runtime: reth_tasks::Runtime, ) -> Self { let precompile_cache_map = PrecompileCacheMap::default(); @@ -1459,7 +1456,7 @@ where changeset_cache: ChangesetCache, ) -> OverlayBuilder { OverlayBuilder::new(parent_hash, changeset_cache) - .with_state_trie_overlay_manager(state.tree_state.state_trie_overlays.clone()) + .with_overlay_manager(state.tree_state.state_trie_overlays.clone()) } /// Prepares the optional payload-builder state-root handle through the installed diff --git a/crates/engine/tree/src/tree/state.rs b/crates/engine/tree/src/tree/state.rs index b8a33adb8b0..75921af45ad 100644 --- a/crates/engine/tree/src/tree/state.rs +++ b/crates/engine/tree/src/tree/state.rs @@ -6,8 +6,9 @@ use alloy_primitives::{ map::{B256Map, B256Set}, BlockNumber, B256, }; -use reth_chain_state::{EthPrimitives, ExecutedBlock, StateTrieOverlayManager}; +use reth_chain_state::{EthPrimitives, ExecutedBlock}; use reth_primitives_traits::{AlloyBlockHeader, NodePrimitives, SealedHeader}; +use reth_storage_overlay::OverlayManager; use std::{ collections::{btree_map, hash_map, BTreeMap, VecDeque}, ops::Bound, @@ -39,7 +40,7 @@ pub struct TreeState { /// The engine API variant of this handler pub(crate) engine_kind: EngineApiKind, /// Flattened state trie overlays for in-memory blocks. - pub(crate) state_trie_overlays: StateTrieOverlayManager, + pub(crate) state_trie_overlays: OverlayManager, } impl TreeState { @@ -47,7 +48,7 @@ impl TreeState { pub fn new( current_canonical_head: BlockNumHash, engine_kind: EngineApiKind, - state_trie_overlays: StateTrieOverlayManager, + state_trie_overlays: OverlayManager, ) -> Self { Self { blocks_by_hash: B256Map::default(), @@ -400,7 +401,7 @@ mod tests { let mut tree_state = TreeState::new( BlockNumHash::default(), EngineApiKind::Ethereum, - StateTrieOverlayManager::default(), + OverlayManager::default(), ); let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); @@ -427,7 +428,7 @@ mod tests { let mut tree_state = TreeState::new( BlockNumHash::default(), EngineApiKind::Ethereum, - StateTrieOverlayManager::default(), + OverlayManager::default(), ); let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); @@ -457,7 +458,7 @@ mod tests { let mut tree_state = TreeState::new( BlockNumHash::default(), EngineApiKind::Ethereum, - StateTrieOverlayManager::default(), + OverlayManager::default(), ); let mut test_block_builder = TestBlockBuilder::eth(); let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..6).collect(); @@ -498,11 +499,8 @@ mod tests { #[tokio::test] async fn test_tree_state_remove_before() { let start_num_hash = BlockNumHash::default(); - let mut tree_state = TreeState::new( - start_num_hash, - EngineApiKind::Ethereum, - StateTrieOverlayManager::default(), - ); + let mut tree_state = + TreeState::new(start_num_hash, EngineApiKind::Ethereum, OverlayManager::default()); let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); for block in &blocks { @@ -552,11 +550,8 @@ mod tests { #[tokio::test] async fn test_tree_state_remove_before_finalized() { let start_num_hash = BlockNumHash::default(); - let mut tree_state = TreeState::new( - start_num_hash, - EngineApiKind::Ethereum, - StateTrieOverlayManager::default(), - ); + let mut tree_state = + TreeState::new(start_num_hash, EngineApiKind::Ethereum, OverlayManager::default()); let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); for block in &blocks { @@ -606,11 +601,8 @@ mod tests { #[tokio::test] async fn test_tree_state_remove_before_lower_finalized() { let start_num_hash = BlockNumHash::default(); - let mut tree_state = TreeState::new( - start_num_hash, - EngineApiKind::Ethereum, - StateTrieOverlayManager::default(), - ); + let mut tree_state = + TreeState::new(start_num_hash, EngineApiKind::Ethereum, OverlayManager::default()); let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..6).collect(); for block in &blocks { diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index f69994e6a8a..13e7c8327ce 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -54,7 +54,7 @@ use crate::tree::{ }; use alloy_primitives::B256; use crossbeam_channel::Receiver as CrossbeamReceiver; -use reth_chain_state::{ExecutedBlock, PreservedSparseTrie, StateTrieOverlayManager}; +use reth_chain_state::{ExecutedBlock, PreservedSparseTrie}; use reth_errors::ProviderResult; use reth_evm::{ConfigureEvm, OnStateHook}; use reth_primitives_traits::{ @@ -65,6 +65,7 @@ use reth_provider::{ DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, StateProviderFactory, StateReader, StateRootProvider, }; +use reth_storage_overlay::OverlayManager; use reth_tasks::utils::increase_thread_priority; use reth_trie::{ hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, @@ -132,7 +133,7 @@ where N: NodePrimitives, { executor: &'a reth_tasks::Runtime, - state_trie_overlays: &'a StateTrieOverlayManager, + state_trie_overlays: &'a OverlayManager, parent_hash: B256, parent_header: &'a N::BlockHeader, timestamp: u64, @@ -164,7 +165,7 @@ where #[expect(clippy::too_many_arguments)] pub(crate) const fn new( executor: &'a reth_tasks::Runtime, - state_trie_overlays: &'a StateTrieOverlayManager, + state_trie_overlays: &'a OverlayManager, parent_hash: B256, parent_header: &'a N::BlockHeader, timestamp: u64, @@ -240,7 +241,7 @@ where Evm: ConfigureEvm, { executor: &'a reth_tasks::Runtime, - state_trie_overlays: &'a StateTrieOverlayManager, + state_trie_overlays: &'a OverlayManager, env: &'a ExecutionEnv, parent_header: &'a SealedHeader, provider_builder: StateProviderBuilder, @@ -272,7 +273,7 @@ where #[expect(clippy::too_many_arguments)] pub(crate) const fn new( executor: &'a reth_tasks::Runtime, - state_trie_overlays: &'a StateTrieOverlayManager, + state_trie_overlays: &'a OverlayManager, env: &'a ExecutionEnv, parent_header: &'a SealedHeader, provider_builder: StateProviderBuilder, @@ -509,7 +510,7 @@ impl DefaultStateRootStrategy { fn spawn_state_root( &self, executor: &reth_tasks::Runtime, - state_trie_overlays: &StateTrieOverlayManager, + state_trie_overlays: &OverlayManager, multiproof_provider_factory: F, options: StateRootTaskOptions<'_, N>, ) -> StateRootHandle @@ -576,7 +577,7 @@ impl DefaultStateRootStrategy { fn spawn_sparse_trie_task( &self, executor: &reth_tasks::Runtime, - state_trie_overlays: &StateTrieOverlayManager, + state_trie_overlays: &OverlayManager, proof_worker_handle: ProofWorkerHandle, state_root_tx: mpsc::Sender>, hashed_state_tx: mpsc::Sender>, @@ -1307,7 +1308,7 @@ mod tests { use alloy_consensus::constants::KECCAK_EMPTY; use alloy_primitives::{map::HashMap, Address, U256}; use rand::Rng; - use reth_chain_state::{test_utils::TestBlockBuilder, StateTrieOverlayManager}; + use reth_chain_state::test_utils::TestBlockBuilder; use reth_chainspec::ChainSpec; use reth_db_common::init::init_genesis; use reth_ethereum_primitives::EthPrimitives; @@ -1315,13 +1316,13 @@ mod tests { use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::{Account, StorageEntry}; use reth_provider::{ - providers::{BlockchainProvider, OverlayBuilder, OverlayStateProviderFactory}, + providers::{BlockchainProvider, OverlayStateProviderFactory}, test_utils::create_test_provider_factory_with_chain_spec, HashingWriter, }; + use reth_storage_overlay::{ChangesetCache, OverlayBuilder, OverlayManager}; use reth_testing_utils::generators; use reth_trie::test_utils::state_root; - use reth_trie_db::ChangesetCache; use revm::state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot, TransactionId}; #[test] @@ -1473,7 +1474,7 @@ mod tests { let provider_factory = BlockchainProvider::new(factory).unwrap(); let env: ExecutionEnv = ExecutionEnv::test_default(); let runtime = reth_tasks::Runtime::test(); - let state_trie_overlays = StateTrieOverlayManager::::default(); + let state_trie_overlays = OverlayManager::::default(); let mut state_root_handle = DefaultStateRootStrategy::default().spawn_state_root( &runtime, &state_trie_overlays, diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index ab9594bc85e..1840e9e3405 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -1113,11 +1113,10 @@ mod tests { use super::*; use alloy_primitives::{keccak256, Address, B256, U256}; use reth_provider::{ - providers::{OverlayBuilder, OverlayStateProviderFactory}, - test_utils::create_test_provider_factory, + providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory, ChainSpecProvider, }; - use reth_trie_db::ChangesetCache; + use reth_storage_overlay::{ChangesetCache, OverlayBuilder}; use reth_trie_parallel::proof_task::ProofTaskCtx; use reth_trie_sparse::ArenaParallelSparseTrie; diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index a2f911c3e56..a9323b8e948 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -7,7 +7,7 @@ use crate::{ PersistTarget, TreeConfig, }, }; -use reth_trie_db::ChangesetCache; +use reth_storage_overlay::{ChangesetCache, OverlayManager}; use alloy_eips::eip1898::BlockWithParent; use alloy_primitives::{ @@ -19,7 +19,7 @@ use alloy_rpc_types_engine::{ ExecutionData, ExecutionPayloadSidecar, ExecutionPayloadV1, ForkchoiceState, }; use assert_matches::assert_matches; -use reth_chain_state::{test_utils::TestBlockBuilder, BlockState, StateTrieOverlayManager}; +use reth_chain_state::{test_utils::TestBlockBuilder, BlockState}; use reth_chainspec::{ChainSpec, HOLESKY, MAINNET}; use reth_engine_primitives::{EngineApiValidator, ForkchoiceStatus, NoopInvalidBlockHook}; use reth_ethereum_consensus::EthBeaconConsensus; @@ -206,8 +206,7 @@ impl TestHarness { let (from_tree_tx, from_tree_rx) = unbounded_channel(); let runtime = reth_tasks::Runtime::test(); - let state_trie_overlays = - StateTrieOverlayManager::new(runtime.state_trie_overlay_worker_pool()); + let state_trie_overlays = OverlayManager::new(runtime.state_trie_overlay_worker_pool()); let header = chain_spec.genesis_header().clone(); let header = SealedHeader::seal_slow(header); @@ -288,7 +287,7 @@ impl TestHarness { parent_hash = hash; } - let state_trie_overlays = StateTrieOverlayManager::default(); + let state_trie_overlays = OverlayManager::default(); for block in &blocks { state_trie_overlays.insert_block(block.clone()); } @@ -442,7 +441,7 @@ impl ValidatorTestHarness { TreeConfig::default(), Box::new(NoopInvalidBlockHook::default()), changeset_cache, - StateTrieOverlayManager::default(), + OverlayManager::default(), reth_tasks::Runtime::test(), ); diff --git a/crates/node/builder/Cargo.toml b/crates/node/builder/Cargo.toml index d489bf18cb6..524266446a9 100644 --- a/crates/node/builder/Cargo.toml +++ b/crates/node/builder/Cargo.toml @@ -49,11 +49,11 @@ reth-rpc-eth-types.workspace = true reth-rpc-layer.workspace = true reth-stages.workspace = true reth-static-file.workspace = true +reth-storage-overlay = { workspace = true, features = ["rayon"] } reth-tasks = { workspace = true, features = ["rayon"] } reth-tokio-util.workspace = true reth-tracing = { workspace = true, features = ["std"] } reth-transaction-pool.workspace = true -reth-trie-db = { workspace = true, features = ["metrics"] } reth-basic-payload-builder.workspace = true reth-node-ethstats.workspace = true @@ -115,7 +115,6 @@ test-utils = [ "reth-db-api/test-utils", "reth-provider/test-utils", "reth-transaction-pool/test-utils", - "reth-trie-db/test-utils", "reth-evm-ethereum/test-utils", "reth-node-ethereum/test-utils", "reth-primitives-traits/test-utils", diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 9650c841086..857bdd4eb6e 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -81,13 +81,13 @@ use reth_stages::{ StageId, StageSet, }; use reth_static_file::{blocks_per_file_for_prune_distance, StaticFileProducer, StaticFileSegment}; +use reth_storage_overlay::ChangesetCache; use reth_tasks::TaskExecutor; use reth_tracing::{ throttle, tracing::{debug, error, info, warn}, }; use reth_transaction_pool::TransactionPool; -use reth_trie_db::ChangesetCache; use std::{num::NonZeroUsize, sync::Arc, thread::available_parallelism, time::Duration}; use tokio::sync::{ mpsc::{unbounded_channel, UnboundedSender}, diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 21e8fcdf40d..57e6737f6a1 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -11,7 +11,6 @@ use crate::{ }; use alloy_consensus::BlockHeader; use futures::{stream::FusedStream, stream_select, FutureExt, StreamExt}; -use reth_chain_state::StateTrieOverlayManager; use reth_chainspec::{EthChainSpec, EthereumHardforks}; use reth_db::{database_metrics::DatabaseMetrics, Database}; use reth_engine_tree::{ @@ -38,10 +37,10 @@ use reth_provider::{ providers::{BlockchainProvider, NodeTypesForProvider}, BlockNumReader, StorageSettingsCache, }; +use reth_storage_overlay::{ChangesetCache, OverlayManager}; use reth_tasks::TaskExecutor; use reth_tokio_util::EventSender; use reth_tracing::tracing::{debug, error, info}; -use reth_trie_db::ChangesetCache; use std::{future::Future, pin::Pin, sync::Arc}; use tokio::sync::{mpsc::unbounded_channel, oneshot}; use tokio_stream::wrappers::UnboundedReceiverStream; @@ -205,7 +204,7 @@ impl EngineNodeLauncher { }; let validator_builder = add_ons.engine_validator_builder(); let state_trie_overlays = - StateTrieOverlayManager::new(ctx.task_executor().state_trie_overlay_worker_pool()); + OverlayManager::new(ctx.task_executor().state_trie_overlay_worker_pool()); // Build the engine validator with all required components let engine_validator = validator_builder diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index 300b9b721c4..bcf0942c7a1 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -10,7 +10,7 @@ pub use reth_rpc_builder::{ middleware::{RethAuthHttpMiddleware, RethRpcMiddleware}, Identity, Stack, }; -pub use reth_trie_db::ChangesetCache; +use reth_storage_overlay::{ChangesetCache, OverlayManager}; use crate::{ invalid_block_hook::InvalidBlockHookExt, txpool_prewarm, ConfigureEngineEvm, @@ -20,7 +20,7 @@ use alloy_rpc_types::engine::ClientVersionV1; use alloy_rpc_types_engine::ExecutionData; use jsonrpsee::RpcModule; use parking_lot::Mutex; -use reth_chain_state::{CanonStateSubscriptions, StateTrieOverlayManager}; +use reth_chain_state::CanonStateSubscriptions; use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks}; use reth_node_api::{ AddOnsContext, BlockTy, EngineApiValidator, EngineTypes, FullNodeComponents, FullNodeTypes, @@ -1413,7 +1413,7 @@ pub trait EngineValidatorBuilder: Send + Sync + Clone ctx: &AddOnsContext<'_, Node>, tree_config: TreeConfig, changeset_cache: ChangesetCache, - state_trie_overlays: StateTrieOverlayManager>, + state_trie_overlays: OverlayManager>, ) -> impl Future> + Send; } @@ -1462,7 +1462,7 @@ where ctx: &AddOnsContext<'_, Node>, tree_config: TreeConfig, changeset_cache: ChangesetCache, - state_trie_overlays: StateTrieOverlayManager>, + state_trie_overlays: OverlayManager>, ) -> eyre::Result { let validator = self.payload_validator_builder.build(ctx).await?; let data_dir = ctx.config.datadir.clone().resolve_datadir(ctx.config.chain.chain()); diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 6b0bd29a52f..1ff298889c1 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -20,6 +20,7 @@ reth-primitives-traits = { workspace = true, features = ["reth-codec", "secp256k reth-errors.workspace = true reth-storage-errors.workspace = true reth-storage-api = { workspace = true, features = ["std", "db-api"] } +reth-storage-overlay.workspace = true reth-db = { workspace = true, features = ["mdbx"] } reth-db-api.workspace = true reth-prune-types.workspace = true @@ -87,7 +88,10 @@ tokio-stream = { workspace = true, features = ["sync"] } [features] jemalloc = ["rocksdb/jemalloc"] -partial-persistence = ["reth-stages-types/partial-persistence"] +partial-persistence = [ + "reth-stages-types/partial-persistence", + "reth-storage-overlay/partial-persistence", +] test-utils = [ "reth-db/test-utils", "reth-nippy-jar/test-utils", diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 5549c825c0f..23b073419f4 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,7 +1,7 @@ use crate::{ providers::{ - ConsistentProvider, OverlayBuilder, OverlayStateProvider, OverlayStateProviderFactory, - ProviderNodeTypes, RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut, + ConsistentProvider, OverlayStateProvider, OverlayStateProviderFactory, ProviderNodeTypes, + RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut, }, AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, @@ -35,6 +35,7 @@ use reth_storage_api::{ StorageRangeResult, }; use reth_storage_errors::provider::ProviderResult; +use reth_storage_overlay::OverlayBuilder; use reth_trie::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, metrics::TrieRootMetrics, diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 9f6785774af..ccd6e59df55 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -33,8 +33,8 @@ use reth_storage_api::{ NodePrimitivesProvider, StorageSettings, StorageSettingsCache, TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::ProviderResult; +use reth_storage_overlay::ChangesetCache; use reth_trie::HashedPostState; -use reth_trie_db::ChangesetCache; use revm::database::BundleState; use std::{ ops::{RangeBounds, RangeInclusive}, diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 03d6235db6e..52bc6210ade 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -65,11 +65,12 @@ use reth_storage_api::{ StoragePath, StorageSettingsCache, TryIntoHistoricalStateProvider, WriteStateInput, }; use reth_storage_errors::provider::{ProviderResult, StaticFileWriterError}; +use reth_storage_overlay::ChangesetCache; use reth_trie::{ updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, ComputedTrieData, HashedPostStateSorted, }; -use reth_trie_db::{ChangesetCache, DatabaseStorageTrieCursor, TrieTableAdapter}; +use reth_trie_db::{DatabaseStorageTrieCursor, TrieTableAdapter}; use revm::database::states::{ PlainStateReverts, PlainStorageChangeset, PlainStorageRevert, StateChangeset, }; diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index 891efc06516..ac6a12fef5c 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -20,7 +20,7 @@ pub use state::{ HistoricalStateProviderRef, HistoryInfo, LowestAvailableBlocks, }, latest::{LatestStateProvider, LatestStateProviderRef}, - overlay::{OverlayBuilder, OverlayStateProvider, OverlayStateProviderFactory}, + overlay::{OverlayStateProvider, OverlayStateProviderFactory}, }; mod consistent_view; diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index e4db4c65295..a2211941b77 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -1,4 +1,3 @@ -use super::overlay::{Overlay, OverlayBuilder, OverlaySource}; use crate::{ AccountReader, BlockHashReader, ChangeSetReader, EitherReader, HashedPostStateProvider, ProviderError, RocksDBProviderFactory, StateProvider, StateRootProvider, @@ -19,6 +18,7 @@ use reth_storage_api::{ StorageSettingsCache, }; use reth_storage_errors::provider::ProviderResult; +use reth_storage_overlay::{ChangesetCache, Overlay, OverlayBuilder, OverlaySource}; use reth_trie::{ hashed_cursor::HashedPostStateCursorFactory, proof::{Proof, StorageProof}, @@ -29,9 +29,7 @@ use reth_trie::{ MultiProof, MultiProofTargets, StateRoot, StorageMultiProof, StorageRoot, TrieInput, TrieInputSorted, }; -use reth_trie_db::{ - ChangesetCache, DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot, -}; +use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot}; use std::{fmt::Debug, marker::PhantomData, sync::Arc}; @@ -891,7 +889,7 @@ mod tests { StorageChangeSetReader, StorageSettingsCache, }; use reth_storage_errors::provider::ProviderError; - use reth_trie_db::ChangesetCache; + use reth_storage_overlay::ChangesetCache; const ADDRESS: Address = address!("0x0000000000000000000000000000000000000001"); const HIGHER_ADDRESS: Address = address!("0x0000000000000000000000000000000000000005"); diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index 27c0c823b2d..c8a4cd487fe 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -1,21 +1,19 @@ -use alloy_eips::BlockNumHash; -use alloy_primitives::{BlockHash, BlockNumber, B256}; +use alloy_primitives::{BlockHash, B256}; use metrics::{Counter, Histogram}; -use reth_chain_state::{EthPrimitives, StateTrieOverlayManager}; use reth_db_api::{tables, transaction::DbTx, DatabaseError}; -use reth_errors::{ProviderError, ProviderResult}; +use reth_errors::ProviderResult; +use reth_ethereum_primitives::EthPrimitives; use reth_metrics::Metrics; use reth_primitives_traits::{ dashmap::{self, DashMap}, NodePrimitives, }; -use reth_prune_types::PruneSegment; -use reth_stages_types::StageId; use reth_storage_api::{ BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, DatabaseProviderROFactory, PruneCheckpointReader, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache, }; +use reth_storage_overlay::{database_state_frontiers, Overlay, OverlayBuilder}; use reth_trie::{ hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory}, trie_cursor::{InMemoryTrieCursor, TrieCursor, TrieCursorFactory, TrieStorageCursor}, @@ -23,487 +21,22 @@ use reth_trie::{ HashedPostStateSorted, }; use reth_trie_db::{ - ChangesetCache, DatabaseAccountTrieCursor, DatabaseHashedCursorFactory, - DatabaseStorageTrieCursor, LegacyKeyAdapter, PackedAccountsTrie, PackedKeyAdapter, - PackedStoragesTrie, + DatabaseAccountTrieCursor, DatabaseHashedCursorFactory, DatabaseStorageTrieCursor, + LegacyKeyAdapter, PackedAccountsTrie, PackedKeyAdapter, PackedStoragesTrie, }; -use std::{ - ops::RangeInclusive, - sync::Arc, - time::{Duration, Instant}, -}; -use tracing::{debug, debug_span, instrument}; +use std::{sync::Arc, time::Instant}; +use tracing::instrument; -/// Metrics for overlay state provider operations. +/// Metrics for overlay state provider factory operations. #[derive(Clone, Metrics)] #[metrics(scope = "storage.providers.overlay")] -pub(crate) struct OverlayStateProviderMetrics { - /// Duration of creating the database provider transaction +pub(crate) struct OverlayStateProviderFactoryMetrics { + /// Duration of creating the database provider transaction. create_provider_duration: Histogram, - /// Duration of retrieving trie updates from the database - retrieve_trie_reverts_duration: Histogram, - /// Duration of retrieving hashed state from the database - retrieve_hashed_state_reverts_duration: Histogram, - /// Size of trie updates (number of entries) - trie_updates_size: Histogram, - /// Size of hashed state (number of entries) - hashed_state_size: Histogram, - /// Overall duration of the [`OverlayStateProviderFactory::database_provider_ro`] call + /// Overall duration of the [`OverlayStateProviderFactory::database_provider_ro`] call. database_provider_ro_duration: Histogram, /// Number of cache misses when fetching [`Overlay`]s from the overlay cache. overlay_cache_misses: Counter, - /// Number of managed overlay creations skipped because the reused sparse trie already covers - /// the DB tip to parent range. - sparse_trie_overlay_skips: Counter, -} - -/// Contains all fields required to initialize an [`OverlayStateProvider`]. -#[derive(Debug, Clone)] -pub(super) struct Overlay { - pub(super) trie_updates: Arc, - pub(super) hashed_post_state: Arc, -} - -impl Overlay { - fn empty() -> Self { - Self { - trie_updates: Arc::new(TrieUpdatesSorted::default()), - hashed_post_state: Arc::new(HashedPostStateSorted::default()), - } - } -} - -/// Source of overlay data for [`OverlayStateProviderFactory`]. -#[derive(Debug, Clone)] -pub(super) enum OverlaySource { - /// Immediate overlay with already-computed data. - Immediate { - /// Trie updates overlay. - /// - /// This can be non-empty when a caller starts with an explicit `TrieInputSorted`, such - /// as historical providers. - trie: Arc, - /// Hashed state overlay. - state: Arc, - }, - /// Manager-backed overlay for in-memory state. - Managed { - /// Manager used to resolve in-memory parent state if the parent is not persisted. - manager: StateTrieOverlayManager, - }, -} - -/// Builder for calculating trie and hashed-state overlays. -/// -/// This stores the overlay configuration and the logic for resolving overlays and collecting -/// reverts. It is intentionally independent from any provider factory or overlay cache. -#[derive(Debug, Clone)] -pub struct OverlayBuilder { - /// Parent hash requested by the caller. - parent_hash: B256, - /// Optional overlay source. - overlay_source: Option>, - /// Changeset cache handle for retrieving trie changesets - changeset_cache: ChangesetCache, - /// Anchor hash of the reused sparse trie, if this task reused one. - reused_sparse_trie_anchor_hash: Option, - /// Metrics for tracking provider operations - metrics: OverlayStateProviderMetrics, -} - -impl OverlayBuilder { - /// Create a new overlay builder. - pub fn new(parent_hash: B256, changeset_cache: ChangesetCache) -> Self { - Self { - parent_hash, - overlay_source: None, - changeset_cache, - reused_sparse_trie_anchor_hash: None, - metrics: OverlayStateProviderMetrics::default(), - } - } - - /// Set the overlay source. - /// - /// This overlay will be applied on top of any reverts. - pub(super) fn with_overlay_source(mut self, source: Option>) -> Self { - self.overlay_source = source; - self - } - - /// Skips managed overlay construction when the sparse trie was reused and the DB tip is - /// already covered by its anchor-to-parent range. - pub const fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { - self.reused_sparse_trie_anchor_hash = Some(anchor_hash); - self - } - - /// Set the state trie overlay manager used to resolve in-memory parent state. - pub fn with_state_trie_overlay_manager( - mut self, - state_trie_overlay_manager: StateTrieOverlayManager, - ) -> Self { - self.overlay_source = Some(OverlaySource::Managed { manager: state_trie_overlay_manager }); - self - } - - /// Set the hashed state overlay. - pub fn with_hashed_state_overlay( - mut self, - hashed_state_overlay: Option>, - ) -> Self { - if let Some(new_state) = hashed_state_overlay { - match &mut self.overlay_source { - Some(OverlaySource::Immediate { state, .. }) => *state = new_state, - Some(OverlaySource::Managed { .. }) | None => { - self.overlay_source = Some(OverlaySource::Immediate { - trie: Arc::new(TrieUpdatesSorted::default()), - state: new_state, - }); - } - } - } - self - } - - /// Set the trie updates overlay. - /// - /// Only applies to an immediate overlay: a managed overlay's trie updates are resolved from - /// its manager instead. - pub fn with_trie_updates_overlay( - mut self, - trie_updates_overlay: Option>, - ) -> Self { - if let Some(trie) = trie_updates_overlay { - match &mut self.overlay_source { - Some(OverlaySource::Immediate { trie: existing, .. }) => *existing = trie, - Some(OverlaySource::Managed { .. }) => {} - None => { - self.overlay_source = Some(OverlaySource::Immediate { - trie, - state: Arc::new(HashedPostStateSorted::default()), - }); - } - } - } - self - } - - /// Resolves the effective overlay (trie updates, hashed state). - fn resolve_overlays( - &self, - anchor_hash: BlockHash, - ) -> ProviderResult<(Arc, Arc)> { - match &self.overlay_source { - Some(OverlaySource::Managed { manager }) => { - if anchor_hash == self.parent_hash { - Ok(( - Arc::new(TrieUpdatesSorted::default()), - Arc::new(HashedPostStateSorted::default()), - )) - } else { - manager - .overlay_for_parent(self.parent_hash, anchor_hash) - .map_err(ProviderError::other) - } - } - Some(OverlaySource::Immediate { trie, state }) => { - if anchor_hash != self.parent_hash { - return Err(ProviderError::other(std::io::Error::other(format!( - "anchor_hash {anchor_hash} doesn't match OverlayBuilder's configured parent ({})", - self.parent_hash - )))) - } - Ok((Arc::clone(trie), Arc::clone(state))) - } - None => Ok(( - Arc::new(TrieUpdatesSorted::default()), - Arc::new(HashedPostStateSorted::default()), - )), - } - } - - /// Returns true if managed overlay resolution can be skipped for this builder. - fn should_skip_overlay_for_reused_sparse_trie( - &self, - state_trie_tip_hash: B256, - finish_tip_hash: B256, - ) -> bool { - let Some(anchor_hash) = self.reused_sparse_trie_anchor_hash else { return false }; - - match &self.overlay_source { - Some(OverlaySource::Managed { manager }) => { - manager.contains_hash(self.parent_hash, anchor_hash, state_trie_tip_hash) && - manager.contains_hash(self.parent_hash, anchor_hash, finish_tip_hash) - } - _ => false, - } - } - - /// Returns whether or not it is required to collect reverts, and validates that there are - /// sufficient changesets to revert to the requested block number if so. - /// - /// Takes into account both the stage checkpoint and the prune checkpoint to determine the - /// available data range. - fn reverts_required( - &self, - provider: &Provider, - state_trie_tip_block: BlockNumHash, - finish_tip_block: BlockNumHash, - anchor_hash: BlockHash, - ) -> ProviderResult>> - where - Provider: BlockNumReader + PruneCheckpointReader, - { - let anchor_number = provider - .convert_hash_or_number(anchor_hash.into())? - .ok_or(ProviderError::BlockHashNotFound(anchor_hash))?; - let canonical_anchor_hash = provider - .convert_number(anchor_number.into())? - .ok_or_else(|| ProviderError::HeaderNotFound(anchor_number.into()))?; - if canonical_anchor_hash != anchor_hash { - return Err(ProviderError::other(std::io::Error::other(format!( - "overlay anchor {anchor_hash} is not on the durable finish chain at block {anchor_number} (found {canonical_anchor_hash})", - )))) - } - - // With no partial-persistence gap, a parent at the Finish tip is already exposed by the - // database without an overlay or reverts. - if state_trie_tip_block.hash == finish_tip_block.hash && - finish_tip_block.hash == anchor_hash - { - return Ok(None) - } - - // The database is a hybrid view while the state/trie and Finish frontiers differ. A - // manager overlay can use that view directly only when its anchor-to-parent path covers - // both frontiers; otherwise the database must first be reverted to the overlay anchor. - if let Some(OverlaySource::Managed { manager }) = &self.overlay_source && - manager.contains_hash(self.parent_hash, anchor_hash, state_trie_tip_block.hash) && - manager.contains_hash(self.parent_hash, anchor_hash, finish_tip_block.hash) - { - return Ok(None) - } - - if anchor_number > state_trie_tip_block.number { - return Err(ProviderError::other(std::io::Error::other(format!( - "overlay anchor #{} ({}) is after partial state trie frontier #{} ({}); missing trie updates for blocks #{}..=#{}", - anchor_number, - anchor_hash, - state_trie_tip_block.number, - state_trie_tip_block.hash, - state_trie_tip_block.number + 1, - anchor_number, - )))) - } - - // Check history prune checkpoints to determine the earliest anchor that can be - // reconstructed. A checkpoint at block N means changesets starting at N + 1 are available, - // which is sufficient to reconstruct the state at N. Both account and storage changesets - // are required, so the later checkpoint determines the lower bound. - let account_history = provider - .get_prune_checkpoint(PruneSegment::AccountHistory)? - .and_then(|checkpoint| checkpoint.block_number); - let storage_history = provider - .get_prune_checkpoint(PruneSegment::StorageHistory)? - .and_then(|checkpoint| checkpoint.block_number); - let lower_bound = account_history.max(storage_history).unwrap_or_default(); - let available_range = lower_bound..=finish_tip_block.number; - if !available_range.contains(&anchor_number) { - return Err(ProviderError::InsufficientChangesets { - requested: anchor_number, - available: available_range, - }) - } - - Ok(Some(anchor_number + 1..=finish_tip_block.number)) - } - - /// Calculates a new [`Overlay`] given a transaction and the current durable frontiers. - #[instrument( - level = "debug", - target = "providers::state::overlay", - skip_all, - fields(?state_trie_tip_block, ?finish_tip_block, parent_hash = ?self.parent_hash) - )] - fn calculate_overlay( - &self, - provider: &Provider, - state_trie_tip_block: BlockNumHash, - finish_tip_block: BlockNumHash, - ) -> ProviderResult - where - Provider: ChangeSetReader - + StorageChangeSetReader - + DBProvider - + BlockNumReader - + StageCheckpointReader - + PruneCheckpointReader - + StorageSettingsCache, - { - // - // Set up variables we'll use for recording metrics. There's two different code-paths here, - // and we want to make sure both record metrics, so we do metrics recording after. - let retrieve_trie_reverts_duration; - let retrieve_hashed_state_reverts_duration; - let trie_updates_total_len; - let hashed_state_updates_total_len; - let anchor_hash = match &self.overlay_source { - Some(OverlaySource::Managed { manager }) => { - let parent_is_persisted = provider - .convert_hash_or_number(self.parent_hash.into())? - .is_some_and(|parent_number| parent_number <= state_trie_tip_block.number); - if parent_is_persisted { - self.parent_hash - } else { - manager - .anchor_for_parent(self.parent_hash, state_trie_tip_block.hash) - .ok_or(ProviderError::BlockHashNotFound(self.parent_hash))? - } - } - _ => self.parent_hash, - }; - - // Collect any reverts which are required to bring the DB view back to the anchor hash. - let (trie_updates, hashed_post_state) = if let Some(revert_blocks) = - self.reverts_required(provider, state_trie_tip_block, finish_tip_block, anchor_hash)? - { - debug!( - target: "providers::state::overlay", - ?revert_blocks, - %anchor_hash, - "Collecting trie reverts for overlay state provider" - ); - - // Collect trie reverts using changeset cache - let trie_reverts = { - let _guard = - debug_span!(target: "providers::state::overlay", "retrieving_trie_reverts") - .entered(); - - let start = Instant::now(); - - // Use changeset cache to retrieve and accumulate reverts to restore state after - // from_block - let accumulated_reverts = - self.changeset_cache.get_or_compute_range(provider, revert_blocks.clone())?; - - retrieve_trie_reverts_duration = start.elapsed(); - accumulated_reverts - }; - - // Collect state reverts - let mut hashed_state_reverts = { - let _guard = debug_span!(target: "providers::state::overlay", "retrieving_hashed_state_reverts").entered(); - - let start = Instant::now(); - let res = reth_trie_db::from_reverts_auto(provider, revert_blocks)?; - retrieve_hashed_state_reverts_duration = start.elapsed(); - res - }; - - // Resolve overlays and extend reverts with them. - // If reverts are empty, use overlays directly to avoid cloning. - let (overlay_trie, overlay_state) = self.resolve_overlays(anchor_hash)?; - - let trie_updates = if trie_reverts.is_empty() { - overlay_trie - } else if !overlay_trie.is_empty() { - let mut trie_reverts = (*trie_reverts).clone(); - trie_reverts.extend_ref_and_sort(&overlay_trie); - Arc::new(trie_reverts) - } else { - trie_reverts - }; - - let hashed_state_updates = if hashed_state_reverts.is_empty() { - overlay_state - } else if !overlay_state.is_empty() { - hashed_state_reverts.extend_ref_and_sort(&overlay_state); - Arc::new(hashed_state_reverts) - } else { - Arc::new(hashed_state_reverts) - }; - - trie_updates_total_len = trie_updates.total_len(); - hashed_state_updates_total_len = hashed_state_updates.total_len(); - - debug!( - target: "providers::state::overlay", - num_trie_updates = ?trie_updates_total_len, - num_state_updates = ?hashed_state_updates_total_len, - %anchor_hash, - "Reverted to anchor block", - ); - - (trie_updates, hashed_state_updates) - } else { - // If no reverts are needed, use the manager overlay directly unless the reused sparse - // trie already covers both durable frontiers through the requested parent. - if self.should_skip_overlay_for_reused_sparse_trie( - state_trie_tip_block.hash, - finish_tip_block.hash, - ) { - debug!( - target: "providers::state::overlay", - parent_hash = %self.parent_hash, - state_trie_tip_hash = %state_trie_tip_block.hash, - finish_tip_hash = %finish_tip_block.hash, - sparse_trie_anchor_hash = ?self.reused_sparse_trie_anchor_hash, - "Skipping overlay construction because reused sparse trie covers durable frontiers to parent" - ); - - self.metrics.sparse_trie_overlay_skips.increment(1); - - return Ok(Overlay::empty()) - } - - let (trie_updates, hashed_post_state) = self.resolve_overlays(anchor_hash)?; - - retrieve_trie_reverts_duration = Duration::ZERO; - retrieve_hashed_state_reverts_duration = Duration::ZERO; - trie_updates_total_len = trie_updates.total_len(); - hashed_state_updates_total_len = hashed_post_state.total_len(); - - debug!( - target: "providers::state::overlay", - num_trie_updates = trie_updates_total_len, - num_state_updates = hashed_state_updates_total_len, - %anchor_hash, - "Built overlay directly from durable frontier" - ); - - (trie_updates, hashed_post_state) - }; - - // Record metrics - self.metrics - .retrieve_trie_reverts_duration - .record(retrieve_trie_reverts_duration.as_secs_f64()); - self.metrics - .retrieve_hashed_state_reverts_duration - .record(retrieve_hashed_state_reverts_duration.as_secs_f64()); - self.metrics.trie_updates_size.record(trie_updates_total_len as f64); - self.metrics.hashed_state_size.record(hashed_state_updates_total_len as f64); - - Ok(Overlay { trie_updates, hashed_post_state }) - } - - /// Builds the effective overlay for the given provider. - #[instrument(level = "debug", target = "providers::state::overlay", skip_all)] - pub(super) fn build_overlay(&self, provider: &Provider) -> ProviderResult - where - Provider: StageCheckpointReader - + PruneCheckpointReader - + ChangeSetReader - + StorageChangeSetReader - + DBProvider - + BlockNumReader - + StorageSettingsCache, - { - let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; - self.calculate_overlay(provider, state_trie_tip_block, finish_tip_block) - } } /// Factory for creating overlay state providers with optional reverts and overlays. @@ -521,12 +54,19 @@ pub struct OverlayStateProviderFactory { /// Under partial persistence the overlay depends on both durable frontiers, so both hashes are /// part of the cache key. overlay_cache: Arc>, + /// Metrics for provider factory operations. + metrics: OverlayStateProviderFactoryMetrics, } impl OverlayStateProviderFactory { /// Create a new overlay state provider factory pub fn new(factory: F, overlay_builder: OverlayBuilder) -> Self { - Self { factory, overlay_builder, overlay_cache: Default::default() } + Self { + factory, + overlay_builder, + overlay_cache: Default::default(), + metrics: Default::default(), + } } /// Set the hashed state overlay. @@ -567,8 +107,12 @@ impl OverlayStateProviderFactory { match self.overlay_cache.entry((state_trie_tip_block.hash, finish_tip_block.hash)) { dashmap::Entry::Occupied(entry) => entry.get().clone(), dashmap::Entry::Vacant(entry) => { - self.overlay_builder.metrics.overlay_cache_misses.increment(1); - let overlay = self.overlay_builder.build_overlay(provider)?; + self.metrics.overlay_cache_misses.increment(1); + let overlay = self.overlay_builder.build_overlay_at_frontiers( + provider, + state_trie_tip_block, + finish_tip_block, + )?; entry.insert(overlay.clone()); overlay } @@ -601,14 +145,14 @@ where let provider = { let start = Instant::now(); let res = self.factory.database_provider_ro()?; - self.overlay_builder.metrics.create_provider_duration.record(start.elapsed()); + self.metrics.create_provider_duration.record(start.elapsed()); res }; let Overlay { trie_updates, hashed_post_state } = self.get_overlay(&provider)?; let is_v2 = provider.cached_storage_settings().is_v2(); - self.overlay_builder.metrics.database_provider_ro_duration.record(overall_start.elapsed()); + self.metrics.database_provider_ro_duration.record(overall_start.elapsed()); Ok(OverlayStateProvider::new(provider, trie_updates, hashed_post_state, is_v2)) } } @@ -731,39 +275,9 @@ where } } -/// Returns the highest blocks whose state/trie data and non-state/trie data are durably -/// available in the database. -pub(crate) fn database_state_frontiers( - provider: &Provider, -) -> ProviderResult<(BlockNumHash, BlockNumHash)> -where - Provider: StageCheckpointReader + BlockNumReader, -{ - let checkpoint = provider - .get_stage_checkpoint(StageId::Finish)? - .ok_or_else(|| ProviderError::InsufficientChangesets { requested: 0, available: 0..=0 })?; - let state_trie_tip_number = checkpoint - .finish_stage_checkpoint() - .and_then(|finish| finish.partial_state_trie()) - .unwrap_or(checkpoint.block_number); - let state_trie_tip_hash = provider - .convert_number(state_trie_tip_number.into())? - .ok_or_else(|| ProviderError::HeaderNotFound(state_trie_tip_number.into()))?; - let finish_tip_number = checkpoint.block_number; - let finish_tip_hash = provider - .convert_number(finish_tip_number.into())? - .ok_or_else(|| ProviderError::HeaderNotFound(finish_tip_number.into()))?; - - Ok(( - BlockNumHash::new(state_trie_tip_number, state_trie_tip_hash), - BlockNumHash::new(finish_tip_number, finish_tip_hash), - )) -} - -#[cfg(test)] +#[cfg(all(test, feature = "partial-persistence"))] mod tests { use super::*; - #[cfg(feature = "partial-persistence")] use crate::{ test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, BlockWriter, ProviderFactory, @@ -771,12 +285,9 @@ mod tests { use alloy_primitives::U256; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; use reth_primitives_traits::Account; - #[cfg(feature = "partial-persistence")] - use reth_prune_types::{PruneCheckpoint, PruneMode}; - #[cfg(feature = "partial-persistence")] - use reth_stages_types::{FinishCheckpoint, StageCheckpoint}; - #[cfg(feature = "partial-persistence")] - use reth_storage_api::{PruneCheckpointWriter, StageCheckpointWriter}; + use reth_stages_types::{FinishCheckpoint, StageCheckpoint, StageId}; + use reth_storage_api::StageCheckpointWriter; + use reth_storage_overlay::{ChangesetCache, OverlayManager}; use reth_trie::{BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles}; fn with_unique_trie_data( @@ -815,7 +326,6 @@ mod tests { .collect() } - #[cfg(feature = "partial-persistence")] fn setup_frontiers( state_trie_tip_index: usize, finish_tip_index: usize, @@ -840,168 +350,18 @@ mod tests { (factory, blocks) } - #[cfg(feature = "partial-persistence")] fn account_keys(overlay: &Overlay) -> Vec { overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() } - #[cfg(feature = "partial-persistence")] fn account_node_paths(overlay: &Overlay) -> Vec { overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() } - #[cfg(feature = "partial-persistence")] - #[test] - fn managed_overlay_starts_at_state_trie_frontier() { - let (factory, blocks) = setup_frontiers(1, 3); - let manager = StateTrieOverlayManager::default(); - for block in &blocks[2..=4] { - manager.insert_block(block.clone()); - } - let provider = factory.provider().unwrap(); - - for (parent_index, expected_ids) in [(3, vec![3, 4]), (4, vec![3, 4, 5])] { - let overlay = OverlayBuilder::::new( - blocks[parent_index].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_state_trie_overlay_manager(manager.clone()) - .build_overlay(&provider) - .unwrap(); - - assert_eq!( - account_keys(&overlay), - expected_ids.iter().copied().map(B256::with_last_byte).collect::>() - ); - assert_eq!( - account_node_paths(&overlay), - expected_ids - .iter() - .copied() - .map(|id| Nibbles::from_nibbles([id])) - .collect::>() - ); - } - } - - #[cfg(feature = "partial-persistence")] - #[test] - fn managed_overlay_uses_persisted_parent_even_if_retained() { - let (factory, blocks) = setup_frontiers(2, 3); - let manager = StateTrieOverlayManager::default(); - manager.insert_block(blocks[1].clone()); - let provider = factory.provider().unwrap(); - - let overlay = OverlayBuilder::::new( - blocks[1].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_state_trie_overlay_manager(manager) - .build_overlay(&provider) - .unwrap(); - - assert!(overlay.hashed_post_state.is_empty()); - assert!(overlay.trie_updates.is_empty()); - } - - #[cfg(feature = "partial-persistence")] - #[test] - fn parent_inside_finish_gap_reverts_to_state_trie_frontier() { - let (factory, blocks) = setup_frontiers(1, 3); - let manager = StateTrieOverlayManager::default(); - manager.insert_block(blocks[2].clone()); - let provider = factory.provider().unwrap(); - let builder = OverlayBuilder::::new( - blocks[2].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_state_trie_overlay_manager(manager); - let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); - let anchor_hash = blocks[1].recovered_block().hash(); - let revert_blocks = - builder.reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash).unwrap(); - - assert_eq!(revert_blocks, Some(2..=3)); - } - - #[cfg(feature = "partial-persistence")] - #[test] - fn anchor_at_prune_checkpoint_has_sufficient_changesets() { - let (factory, blocks) = setup_frontiers(1, 3); - let provider_rw = factory.provider_rw().unwrap(); - provider_rw - .save_prune_checkpoint( - PruneSegment::AccountHistory, - PruneCheckpoint { - block_number: Some(blocks[1].block_number()), - tx_number: None, - prune_mode: PruneMode::Full, - }, - ) - .unwrap(); - provider_rw.commit().unwrap(); - - let manager = StateTrieOverlayManager::default(); - manager.insert_block(blocks[2].clone()); - let provider = factory.provider().unwrap(); - let builder = OverlayBuilder::::new( - blocks[2].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_state_trie_overlay_manager(manager); - let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); - let anchor_hash = blocks[1].recovered_block().hash(); - let revert_blocks = - builder.reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash).unwrap(); - - assert_eq!(revert_blocks, Some(2..=3)); - } - - #[cfg(feature = "partial-persistence")] - #[test] - fn storage_history_checkpoint_limits_available_anchor() { - let (factory, blocks) = setup_frontiers(1, 3); - let provider_rw = factory.provider_rw().unwrap(); - provider_rw - .save_prune_checkpoint( - PruneSegment::StorageHistory, - PruneCheckpoint { - block_number: Some(blocks[2].block_number()), - tx_number: None, - prune_mode: PruneMode::Full, - }, - ) - .unwrap(); - provider_rw.commit().unwrap(); - - let manager = StateTrieOverlayManager::default(); - manager.insert_block(blocks[2].clone()); - let provider = factory.provider().unwrap(); - let builder = OverlayBuilder::::new( - blocks[2].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_state_trie_overlay_manager(manager); - let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); - let anchor_hash = blocks[1].recovered_block().hash(); - let error = builder - .reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash) - .unwrap_err(); - - match error { - ProviderError::InsufficientChangesets { requested, available } => { - assert_eq!(requested, blocks[1].block_number()); - assert_eq!(available, blocks[2].block_number()..=blocks[3].block_number()); - } - error => panic!("unexpected error: {error}"), - } - } - - #[cfg(feature = "partial-persistence")] #[test] fn overlay_cache_is_keyed_by_both_durable_frontiers() { let (factory, blocks) = setup_frontiers(1, 3); - let manager = StateTrieOverlayManager::default(); + let manager = OverlayManager::default(); for block in &blocks[2..=3] { manager.insert_block(block.clone()); } @@ -1011,7 +371,7 @@ mod tests { blocks[3].recovered_block().hash(), ChangesetCache::new(), ) - .with_state_trie_overlay_manager(manager), + .with_overlay_manager(manager), ); let provider = factory.provider().unwrap(); @@ -1036,95 +396,4 @@ mod tests { assert_eq!(account_node_paths(&second), vec![Nibbles::from_nibbles([4])]); assert_eq!(overlay_factory.overlay_cache.len(), 2); } - - #[cfg(feature = "partial-persistence")] - #[test] - fn overlay_after_state_trie_frontier_requires_managed_coverage() { - let (factory, blocks) = setup_frontiers(1, 3); - let provider = factory.provider().unwrap(); - let error = OverlayBuilder::::new( - blocks[3].recovered_block().hash(), - ChangesetCache::new(), - ) - .build_overlay(&provider) - .unwrap_err(); - - assert!( - error.to_string().contains("is after partial state trie frontier"), - "unexpected error: {error}" - ); - } - - #[cfg(feature = "partial-persistence")] - #[test] - fn managed_overlay_errors_if_parent_is_not_persisted_or_managed_across_frontiers() { - let (factory, blocks) = setup_frontiers(1, 3); - let provider = factory.provider().unwrap(); - let parent_hash = blocks[3].recovered_block().hash(); - let error = OverlayBuilder::::new(parent_hash, ChangesetCache::new()) - .with_state_trie_overlay_manager(StateTrieOverlayManager::default()) - .build_overlay(&provider) - .unwrap_err(); - - assert!(matches!(error, ProviderError::BlockHashNotFound(hash) if hash == parent_hash)); - } - - #[test] - fn managed_overlay_skips_manager_for_persisted_parent() { - let parent_hash = B256::with_last_byte(1); - let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) - .with_state_trie_overlay_manager(StateTrieOverlayManager::default()); - - let (trie, state) = builder.resolve_overlays(parent_hash).unwrap(); - assert!(trie.is_empty()); - assert!(state.is_empty()); - } - - #[test] - fn managed_overlay_errors_if_parent_is_not_persisted_or_managed() { - let parent_hash = B256::with_last_byte(1); - let anchor_hash = B256::with_last_byte(2); - let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) - .with_state_trie_overlay_manager(StateTrieOverlayManager::default()); - - let err = builder.resolve_overlays(anchor_hash).unwrap_err(); - - assert!(err.to_string().contains("cannot be anchored")); - } - - #[test] - fn managed_overlay_skip_requires_both_frontiers() { - let parent_hash = B256::with_last_byte(1); - let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) - .with_state_trie_overlay_manager(StateTrieOverlayManager::default()); - assert!(!builder.should_skip_overlay_for_reused_sparse_trie(parent_hash, parent_hash)); - - let builder = builder.with_skip_overlay_for_reused_sparse_trie(parent_hash); - assert!(builder.should_skip_overlay_for_reused_sparse_trie(parent_hash, parent_hash)); - assert!(!builder - .should_skip_overlay_for_reused_sparse_trie(B256::with_last_byte(3), parent_hash,)); - - let blocks = test_blocks(); - let manager = StateTrieOverlayManager::default(); - for block in &blocks[2..=4] { - manager.insert_block(block.clone()); - } - let builder = OverlayBuilder::::new( - blocks[4].recovered_block().hash(), - ChangesetCache::default(), - ) - .with_state_trie_overlay_manager(manager) - .with_skip_overlay_for_reused_sparse_trie(blocks[1].recovered_block().hash()); - assert!(builder.should_skip_overlay_for_reused_sparse_trie( - blocks[1].recovered_block().hash(), - blocks[3].recovered_block().hash(), - )); - - let builder = - builder.with_skip_overlay_for_reused_sparse_trie(blocks[2].recovered_block().hash()); - assert!(!builder.should_skip_overlay_for_reused_sparse_trie( - blocks[1].recovered_block().hash(), - blocks[3].recovered_block().hash(), - )); - } } diff --git a/crates/storage/storage-overlay/Cargo.toml b/crates/storage/storage-overlay/Cargo.toml new file mode 100644 index 00000000000..f9a6e0a2b13 --- /dev/null +++ b/crates/storage/storage-overlay/Cargo.toml @@ -0,0 +1,53 @@ +[package] +name = "reth-storage-overlay" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +description = "State trie overlay construction and caching" + +[lints] +workspace = true + +[dependencies] +# reth +reth-chain-state.workspace = true +reth-errors.workspace = true +reth-ethereum-primitives.workspace = true +reth-metrics.workspace = true +reth-primitives-traits = { workspace = true, features = ["dashmap", "std"] } +reth-prune-types.workspace = true +reth-stages-types.workspace = true +reth-storage-api = { workspace = true, features = ["db-api"] } +reth-storage-errors.workspace = true +reth-tasks = { workspace = true, features = ["rayon"], optional = true } +reth-trie.workspace = true +reth-trie-common.workspace = true +reth-trie-db.workspace = true + +# alloy +alloy-eips.workspace = true +alloy-primitives.workspace = true + +# metrics +metrics.workspace = true + +# misc +parking_lot.workspace = true +rayon = { workspace = true, optional = true } +tracing.workspace = true + +[dev-dependencies] +reth-chain-state = { workspace = true, features = ["test-utils"] } +reth-db = { workspace = true, features = ["test-utils"] } +reth-primitives-traits = { workspace = true, features = ["test-utils"] } +reth-provider = { workspace = true, features = ["test-utils"] } +reth-trie = { workspace = true, features = ["test-utils"] } + +alloy-consensus.workspace = true + +[features] +partial-persistence = ["reth-stages-types/partial-persistence"] +rayon = ["dep:rayon", "dep:reth-tasks"] diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs new file mode 100644 index 00000000000..80e4fb39622 --- /dev/null +++ b/crates/storage/storage-overlay/src/builder.rs @@ -0,0 +1,835 @@ +use crate::{ChangesetCache, OverlayManager}; +use alloy_eips::BlockNumHash; +use alloy_primitives::{BlockHash, BlockNumber, B256}; +use metrics::{Counter, Histogram}; +use reth_errors::{ProviderError, ProviderResult}; +use reth_ethereum_primitives::EthPrimitives; +use reth_metrics::Metrics; +use reth_primitives_traits::NodePrimitives; +use reth_prune_types::PruneSegment; +use reth_stages_types::StageId; +use reth_storage_api::{ + BlockNumReader, ChangeSetReader, DBProvider, PruneCheckpointReader, StageCheckpointReader, + StorageChangeSetReader, StorageSettingsCache, +}; +use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; +use std::{ + ops::RangeInclusive, + sync::Arc, + time::{Duration, Instant}, +}; +use tracing::{debug, debug_span, instrument}; + +/// Contains the trie and hashed-state data required to initialize an overlay state provider. +#[derive(Debug, Clone)] +pub struct Overlay { + /// Trie updates overlay. + pub trie_updates: Arc, + /// Hashed state overlay. + pub hashed_post_state: Arc, +} + +impl Overlay { + fn empty() -> Self { + Self { + trie_updates: Arc::new(TrieUpdatesSorted::default()), + hashed_post_state: Arc::new(HashedPostStateSorted::default()), + } + } +} + +/// Source of data to apply on top of the durable database state. +#[derive(Debug, Clone)] +pub enum OverlaySource { + /// Immediate overlay with already-computed data. + Immediate { + /// Trie updates overlay. + /// + /// This can be non-empty when a caller starts with an explicit `TrieInputSorted`, such + /// as historical providers. + trie: Arc, + /// Hashed state overlay. + state: Arc, + }, + /// Manager-backed overlay for in-memory state. + Managed { + /// Manager used to resolve in-memory parent state if the parent is not persisted. + manager: OverlayManager, + }, +} + +/// Builder for calculating trie and hashed-state overlays. +/// +/// This stores the overlay configuration and the logic for resolving overlays and collecting +/// reverts. It is independent from any provider factory or overlay cache. +#[derive(Debug, Clone)] +pub struct OverlayBuilder { + /// Parent hash requested by the caller. + parent_hash: B256, + /// Optional overlay source. + overlay_source: Option>, + /// Changeset cache handle for retrieving trie changesets. + changeset_cache: ChangesetCache, + /// Anchor hash of the reused sparse trie, if this task reused one. + reused_sparse_trie_anchor_hash: Option, + /// Metrics for overlay construction. + metrics: OverlayBuilderMetrics, +} + +impl OverlayBuilder { + /// Create a new overlay builder. + pub fn new(parent_hash: B256, changeset_cache: ChangesetCache) -> Self { + Self { + parent_hash, + overlay_source: None, + changeset_cache, + reused_sparse_trie_anchor_hash: None, + metrics: OverlayBuilderMetrics::default(), + } + } + + /// Set the overlay source. + /// + /// This overlay will be applied on top of any reverts. + pub fn with_overlay_source(mut self, source: Option>) -> Self { + self.overlay_source = source; + self + } + + /// Skips managed overlay construction when the sparse trie was reused and the DB tip is + /// already covered by its anchor-to-parent range. + pub const fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { + self.reused_sparse_trie_anchor_hash = Some(anchor_hash); + self + } + + /// Set the overlay manager used to resolve in-memory parent state. + pub fn with_overlay_manager(mut self, overlay_manager: OverlayManager) -> Self { + self.overlay_source = Some(OverlaySource::Managed { manager: overlay_manager }); + self + } + + /// Set the hashed state overlay. + pub fn with_hashed_state_overlay( + mut self, + hashed_state_overlay: Option>, + ) -> Self { + if let Some(new_state) = hashed_state_overlay { + match &mut self.overlay_source { + Some(OverlaySource::Immediate { state, .. }) => *state = new_state, + Some(OverlaySource::Managed { .. }) | None => { + self.overlay_source = Some(OverlaySource::Immediate { + trie: Arc::new(TrieUpdatesSorted::default()), + state: new_state, + }); + } + } + } + self + } + + /// Set the trie updates overlay. + /// + /// Only applies to an immediate overlay: a managed overlay's trie updates are resolved from + /// its manager instead. + pub fn with_trie_updates_overlay( + mut self, + trie_updates_overlay: Option>, + ) -> Self { + if let Some(trie) = trie_updates_overlay { + match &mut self.overlay_source { + Some(OverlaySource::Immediate { trie: existing, .. }) => *existing = trie, + Some(OverlaySource::Managed { .. }) => {} + None => { + self.overlay_source = Some(OverlaySource::Immediate { + trie, + state: Arc::new(HashedPostStateSorted::default()), + }); + } + } + } + self + } + + /// Builds the effective overlay for the given provider. + #[instrument(level = "debug", target = "storage::overlay", skip_all)] + pub fn build_overlay(&self, provider: &Provider) -> ProviderResult + where + Provider: StageCheckpointReader + + PruneCheckpointReader + + ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader + + StorageSettingsCache, + { + let (state_trie_tip_block, finish_tip_block) = database_state_frontiers(provider)?; + self.build_overlay_at_frontiers(provider, state_trie_tip_block, finish_tip_block) + } + + /// Builds the effective overlay using frontiers already read from the provider. + /// + /// This is useful for callers that key an overlay cache by the durable frontiers. + #[instrument( + level = "debug", + target = "storage::overlay", + skip_all, + fields(?state_trie_tip_block, ?finish_tip_block, parent_hash = ?self.parent_hash) + )] + pub fn build_overlay_at_frontiers( + &self, + provider: &Provider, + state_trie_tip_block: BlockNumHash, + finish_tip_block: BlockNumHash, + ) -> ProviderResult + where + Provider: ChangeSetReader + + StorageChangeSetReader + + DBProvider + + BlockNumReader + + StageCheckpointReader + + PruneCheckpointReader + + StorageSettingsCache, + { + let retrieve_trie_reverts_duration; + let retrieve_hashed_state_reverts_duration; + let trie_updates_total_len; + let hashed_state_updates_total_len; + let anchor_hash = match &self.overlay_source { + Some(OverlaySource::Managed { manager }) => { + let parent_is_persisted = provider + .convert_hash_or_number(self.parent_hash.into())? + .is_some_and(|parent_number| parent_number <= state_trie_tip_block.number); + if parent_is_persisted { + self.parent_hash + } else { + manager + .anchor_for_parent(self.parent_hash, state_trie_tip_block.hash) + .ok_or(ProviderError::BlockHashNotFound(self.parent_hash))? + } + } + _ => self.parent_hash, + }; + + // Collect any reverts which are required to bring the DB view back to the anchor hash. + let (trie_updates, hashed_post_state) = if let Some(revert_blocks) = + self.reverts_required(provider, state_trie_tip_block, finish_tip_block, anchor_hash)? + { + debug!( + target: "storage::overlay", + ?revert_blocks, + %anchor_hash, + "Collecting trie reverts for overlay state provider" + ); + + let trie_reverts = { + let _guard = + debug_span!(target: "storage::overlay", "retrieving_trie_reverts").entered(); + let start = Instant::now(); + let accumulated_reverts = + self.changeset_cache.get_or_compute_range(provider, revert_blocks.clone())?; + retrieve_trie_reverts_duration = start.elapsed(); + accumulated_reverts + }; + + let mut hashed_state_reverts = { + let _guard = + debug_span!(target: "storage::overlay", "retrieving_hashed_state_reverts") + .entered(); + let start = Instant::now(); + let res = reth_trie_db::from_reverts_auto(provider, revert_blocks)?; + retrieve_hashed_state_reverts_duration = start.elapsed(); + res + }; + + // Resolve overlays and extend reverts with them. If reverts are empty, use overlays + // directly to avoid cloning. + let (overlay_trie, overlay_state) = self.resolve_overlays(anchor_hash)?; + + let trie_updates = if trie_reverts.is_empty() { + overlay_trie + } else if !overlay_trie.is_empty() { + let mut trie_reverts = (*trie_reverts).clone(); + trie_reverts.extend_ref_and_sort(&overlay_trie); + Arc::new(trie_reverts) + } else { + trie_reverts + }; + + let hashed_state_updates = if hashed_state_reverts.is_empty() { + overlay_state + } else if !overlay_state.is_empty() { + hashed_state_reverts.extend_ref_and_sort(&overlay_state); + Arc::new(hashed_state_reverts) + } else { + Arc::new(hashed_state_reverts) + }; + + trie_updates_total_len = trie_updates.total_len(); + hashed_state_updates_total_len = hashed_state_updates.total_len(); + + debug!( + target: "storage::overlay", + num_trie_updates = ?trie_updates_total_len, + num_state_updates = ?hashed_state_updates_total_len, + %anchor_hash, + "Reverted to anchor block", + ); + + (trie_updates, hashed_state_updates) + } else { + // If no reverts are needed, use the manager overlay directly unless the reused sparse + // trie already covers both durable frontiers through the requested parent. + if self.should_skip_overlay_for_reused_sparse_trie( + state_trie_tip_block.hash, + finish_tip_block.hash, + ) { + debug!( + target: "storage::overlay", + parent_hash = %self.parent_hash, + state_trie_tip_hash = %state_trie_tip_block.hash, + finish_tip_hash = %finish_tip_block.hash, + sparse_trie_anchor_hash = ?self.reused_sparse_trie_anchor_hash, + "Skipping overlay construction because reused sparse trie covers durable frontiers to parent" + ); + + self.metrics.sparse_trie_overlay_skips.increment(1); + + return Ok(Overlay::empty()) + } + + let (trie_updates, hashed_post_state) = self.resolve_overlays(anchor_hash)?; + + retrieve_trie_reverts_duration = Duration::ZERO; + retrieve_hashed_state_reverts_duration = Duration::ZERO; + trie_updates_total_len = trie_updates.total_len(); + hashed_state_updates_total_len = hashed_post_state.total_len(); + + debug!( + target: "storage::overlay", + num_trie_updates = trie_updates_total_len, + num_state_updates = hashed_state_updates_total_len, + %anchor_hash, + "Built overlay directly from durable frontier" + ); + + (trie_updates, hashed_post_state) + }; + + self.metrics + .retrieve_trie_reverts_duration + .record(retrieve_trie_reverts_duration.as_secs_f64()); + self.metrics + .retrieve_hashed_state_reverts_duration + .record(retrieve_hashed_state_reverts_duration.as_secs_f64()); + self.metrics.trie_updates_size.record(trie_updates_total_len as f64); + self.metrics.hashed_state_size.record(hashed_state_updates_total_len as f64); + + Ok(Overlay { trie_updates, hashed_post_state }) + } + + /// Resolves the effective overlay (trie updates, hashed state). + fn resolve_overlays( + &self, + anchor_hash: BlockHash, + ) -> ProviderResult<(Arc, Arc)> { + match &self.overlay_source { + Some(OverlaySource::Managed { manager }) => { + if anchor_hash == self.parent_hash { + Ok(( + Arc::new(TrieUpdatesSorted::default()), + Arc::new(HashedPostStateSorted::default()), + )) + } else { + manager + .overlay_for_parent(self.parent_hash, anchor_hash) + .map_err(ProviderError::other) + } + } + Some(OverlaySource::Immediate { trie, state }) => { + if anchor_hash != self.parent_hash { + return Err(ProviderError::other(std::io::Error::other(format!( + "anchor_hash {anchor_hash} doesn't match OverlayBuilder's configured parent ({})", + self.parent_hash + )))) + } + Ok((Arc::clone(trie), Arc::clone(state))) + } + None => Ok(( + Arc::new(TrieUpdatesSorted::default()), + Arc::new(HashedPostStateSorted::default()), + )), + } + } + + /// Returns true if managed overlay resolution can be skipped for this builder. + fn should_skip_overlay_for_reused_sparse_trie( + &self, + state_trie_tip_hash: B256, + finish_tip_hash: B256, + ) -> bool { + let Some(anchor_hash) = self.reused_sparse_trie_anchor_hash else { return false }; + + match &self.overlay_source { + Some(OverlaySource::Managed { manager }) => { + manager.contains_hash(self.parent_hash, anchor_hash, state_trie_tip_hash) && + manager.contains_hash(self.parent_hash, anchor_hash, finish_tip_hash) + } + _ => false, + } + } + + /// Returns whether or not it is required to collect reverts, and validates that there are + /// sufficient changesets to revert to the requested block number if so. + /// + /// Takes into account both the stage checkpoint and the prune checkpoint to determine the + /// available data range. + fn reverts_required( + &self, + provider: &Provider, + state_trie_tip_block: BlockNumHash, + finish_tip_block: BlockNumHash, + anchor_hash: BlockHash, + ) -> ProviderResult>> + where + Provider: BlockNumReader + PruneCheckpointReader, + { + let anchor_number = provider + .convert_hash_or_number(anchor_hash.into())? + .ok_or(ProviderError::BlockHashNotFound(anchor_hash))?; + let canonical_anchor_hash = provider + .convert_number(anchor_number.into())? + .ok_or_else(|| ProviderError::HeaderNotFound(anchor_number.into()))?; + if canonical_anchor_hash != anchor_hash { + return Err(ProviderError::other(std::io::Error::other(format!( + "overlay anchor {anchor_hash} is not on the durable finish chain at block {anchor_number} (found {canonical_anchor_hash})", + )))) + } + + // With no partial-persistence gap, a parent at the Finish tip is already exposed by the + // database without an overlay or reverts. + if state_trie_tip_block.hash == finish_tip_block.hash && + finish_tip_block.hash == anchor_hash + { + return Ok(None) + } + + // The database is a hybrid view while the state/trie and Finish frontiers differ. A + // manager overlay can use that view directly only when its anchor-to-parent path covers + // both frontiers; otherwise the database must first be reverted to the overlay anchor. + if let Some(OverlaySource::Managed { manager }) = &self.overlay_source && + manager.contains_hash(self.parent_hash, anchor_hash, state_trie_tip_block.hash) && + manager.contains_hash(self.parent_hash, anchor_hash, finish_tip_block.hash) + { + return Ok(None) + } + + if anchor_number > state_trie_tip_block.number { + return Err(ProviderError::other(std::io::Error::other(format!( + "overlay anchor #{} ({}) is after partial state trie frontier #{} ({}); missing trie updates for blocks #{}..=#{}", + anchor_number, + anchor_hash, + state_trie_tip_block.number, + state_trie_tip_block.hash, + state_trie_tip_block.number + 1, + anchor_number, + )))) + } + + // Check history prune checkpoints to determine the earliest anchor that can be + // reconstructed. A checkpoint at block N means changesets starting at N + 1 are available, + // which is sufficient to reconstruct the state at N. Both account and storage changesets + // are required, so the later checkpoint determines the lower bound. + let account_history = provider + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| checkpoint.block_number); + let storage_history = provider + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|checkpoint| checkpoint.block_number); + let lower_bound = account_history.max(storage_history).unwrap_or_default(); + let available_range = lower_bound..=finish_tip_block.number; + if !available_range.contains(&anchor_number) { + return Err(ProviderError::InsufficientChangesets { + requested: anchor_number, + available: available_range, + }) + } + + Ok(Some(anchor_number + 1..=finish_tip_block.number)) + } +} + +/// Returns the highest blocks whose state/trie data and non-state/trie data are durably +/// available in the database. +pub fn database_state_frontiers( + provider: &Provider, +) -> ProviderResult<(BlockNumHash, BlockNumHash)> +where + Provider: StageCheckpointReader + BlockNumReader, +{ + let checkpoint = provider + .get_stage_checkpoint(StageId::Finish)? + .ok_or_else(|| ProviderError::InsufficientChangesets { requested: 0, available: 0..=0 })?; + let state_trie_tip_number = checkpoint + .finish_stage_checkpoint() + .and_then(|finish| finish.partial_state_trie()) + .unwrap_or(checkpoint.block_number); + let state_trie_tip_hash = provider + .convert_number(state_trie_tip_number.into())? + .ok_or_else(|| ProviderError::HeaderNotFound(state_trie_tip_number.into()))?; + let finish_tip_number = checkpoint.block_number; + let finish_tip_hash = provider + .convert_number(finish_tip_number.into())? + .ok_or_else(|| ProviderError::HeaderNotFound(finish_tip_number.into()))?; + + Ok(( + BlockNumHash::new(state_trie_tip_number, state_trie_tip_hash), + BlockNumHash::new(finish_tip_number, finish_tip_hash), + )) +} + +/// Metrics for overlay construction. +#[derive(Clone, Metrics)] +#[metrics(scope = "storage.overlay.builder")] +struct OverlayBuilderMetrics { + /// Duration of retrieving trie updates from the database. + retrieve_trie_reverts_duration: Histogram, + /// Duration of retrieving hashed state from the database. + retrieve_hashed_state_reverts_duration: Histogram, + /// Size of trie updates (number of entries). + trie_updates_size: Histogram, + /// Size of hashed state (number of entries). + hashed_state_size: Histogram, + /// Number of managed overlay creations skipped because the reused sparse trie already covers + /// the DB tip to parent range. + sparse_trie_overlay_skips: Counter, +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::U256; + use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; + use reth_primitives_traits::Account; + #[cfg(feature = "partial-persistence")] + use reth_provider::{ + test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, + BlockWriter, ProviderFactory, + }; + #[cfg(feature = "partial-persistence")] + use reth_prune_types::{PruneCheckpoint, PruneMode}; + #[cfg(feature = "partial-persistence")] + use reth_stages_types::{FinishCheckpoint, StageCheckpoint}; + #[cfg(feature = "partial-persistence")] + use reth_storage_api::{PruneCheckpointWriter, StageCheckpointWriter}; + use reth_trie::{BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles}; + + fn with_unique_trie_data( + block: &ExecutedBlock, + id: u8, + ) -> ExecutedBlock { + let hashed_address = B256::with_last_byte(id); + let hashed_slot = B256::with_last_byte(id.saturating_add(32)); + let hashed_state = HashedPostState::default() + .with_accounts([(hashed_address, Some(Account::default()))]) + .with_storages([( + hashed_address, + HashedStorage::from_iter(false, [(hashed_slot, U256::from(id))]), + )]) + .into_sorted(); + let trie_updates = TrieUpdatesSorted::new( + vec![( + Nibbles::from_nibbles([id]), + Some(BranchNodeCompact::new(0, 0, 0, vec![], None)), + )], + Default::default(), + ); + + ExecutedBlock::new( + Arc::clone(&block.recovered_block), + Arc::clone(&block.execution_output), + ComputedTrieData::new(Arc::new(hashed_state), Arc::new(trie_updates)), + ) + } + + fn test_blocks() -> Vec> { + TestBlockBuilder::eth() + .get_executed_blocks(0..5) + .enumerate() + .map(|(index, block)| with_unique_trie_data(&block, index as u8 + 1)) + .collect() + } + + #[cfg(feature = "partial-persistence")] + fn setup_frontiers( + state_trie_tip_index: usize, + finish_tip_index: usize, + ) -> (ProviderFactory, Vec>) { + let factory = create_test_provider_factory(); + let blocks = test_blocks(); + let provider_rw = factory.provider_rw().unwrap(); + for block in &blocks[..=finish_tip_index] { + provider_rw.insert_block(block.recovered_block()).unwrap(); + } + provider_rw + .save_stage_checkpoint( + StageId::Finish, + StageCheckpoint::new(blocks[finish_tip_index].block_number()) + .with_finish_stage_checkpoint(FinishCheckpoint { + partial_state_trie: Some(blocks[state_trie_tip_index].block_number()), + }), + ) + .unwrap(); + provider_rw.commit().unwrap(); + + (factory, blocks) + } + + #[cfg(feature = "partial-persistence")] + fn account_keys(overlay: &Overlay) -> Vec { + overlay.hashed_post_state.accounts.iter().map(|(key, _)| *key).collect() + } + + #[cfg(feature = "partial-persistence")] + fn account_node_paths(overlay: &Overlay) -> Vec { + overlay.trie_updates.account_nodes_ref().iter().map(|(path, _)| *path).collect() + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn managed_overlay_starts_at_state_trie_frontier() { + let (factory, blocks) = setup_frontiers(1, 3); + let manager = OverlayManager::default(); + for block in &blocks[2..=4] { + manager.insert_block(block.clone()); + } + let provider = factory.provider().unwrap(); + + for (parent_index, expected_ids) in [(3, vec![3, 4]), (4, vec![3, 4, 5])] { + let overlay = OverlayBuilder::::new( + blocks[parent_index].recovered_block().hash(), + ChangesetCache::new(), + ) + .with_overlay_manager(manager.clone()) + .build_overlay(&provider) + .unwrap(); + + assert_eq!( + account_keys(&overlay), + expected_ids.iter().copied().map(B256::with_last_byte).collect::>() + ); + assert_eq!( + account_node_paths(&overlay), + expected_ids + .iter() + .copied() + .map(|id| Nibbles::from_nibbles([id])) + .collect::>() + ); + } + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn managed_overlay_uses_persisted_parent_even_if_retained() { + let (factory, blocks) = setup_frontiers(2, 3); + let manager = OverlayManager::default(); + manager.insert_block(blocks[1].clone()); + let provider = factory.provider().unwrap(); + + let overlay = OverlayBuilder::::new( + blocks[1].recovered_block().hash(), + ChangesetCache::new(), + ) + .with_overlay_manager(manager) + .build_overlay(&provider) + .unwrap(); + + assert!(overlay.hashed_post_state.is_empty()); + assert!(overlay.trie_updates.is_empty()); + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn parent_inside_finish_gap_reverts_to_state_trie_frontier() { + let (factory, blocks) = setup_frontiers(1, 3); + let manager = OverlayManager::default(); + manager.insert_block(blocks[2].clone()); + let provider = factory.provider().unwrap(); + let builder = OverlayBuilder::::new( + blocks[2].recovered_block().hash(), + ChangesetCache::new(), + ) + .with_overlay_manager(manager); + let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); + let anchor_hash = blocks[1].recovered_block().hash(); + let revert_blocks = + builder.reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash).unwrap(); + + assert_eq!(revert_blocks, Some(2..=3)); + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn anchor_at_prune_checkpoint_has_sufficient_changesets() { + let (factory, blocks) = setup_frontiers(1, 3); + let provider_rw = factory.provider_rw().unwrap(); + provider_rw + .save_prune_checkpoint( + PruneSegment::AccountHistory, + PruneCheckpoint { + block_number: Some(blocks[1].block_number()), + tx_number: None, + prune_mode: PruneMode::Full, + }, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let manager = OverlayManager::default(); + manager.insert_block(blocks[2].clone()); + let provider = factory.provider().unwrap(); + let builder = OverlayBuilder::::new( + blocks[2].recovered_block().hash(), + ChangesetCache::new(), + ) + .with_overlay_manager(manager); + let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); + let anchor_hash = blocks[1].recovered_block().hash(); + let revert_blocks = + builder.reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash).unwrap(); + + assert_eq!(revert_blocks, Some(2..=3)); + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn storage_history_checkpoint_limits_available_anchor() { + let (factory, blocks) = setup_frontiers(1, 3); + let provider_rw = factory.provider_rw().unwrap(); + provider_rw + .save_prune_checkpoint( + PruneSegment::StorageHistory, + PruneCheckpoint { + block_number: Some(blocks[2].block_number()), + tx_number: None, + prune_mode: PruneMode::Full, + }, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let manager = OverlayManager::default(); + manager.insert_block(blocks[2].clone()); + let provider = factory.provider().unwrap(); + let builder = OverlayBuilder::::new( + blocks[2].recovered_block().hash(), + ChangesetCache::new(), + ) + .with_overlay_manager(manager); + let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); + let anchor_hash = blocks[1].recovered_block().hash(); + let error = builder + .reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash) + .unwrap_err(); + + match error { + ProviderError::InsufficientChangesets { requested, available } => { + assert_eq!(requested, blocks[1].block_number()); + assert_eq!(available, blocks[2].block_number()..=blocks[3].block_number()); + } + error => panic!("unexpected error: {error}"), + } + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn overlay_after_state_trie_frontier_requires_managed_coverage() { + let (factory, blocks) = setup_frontiers(1, 3); + let provider = factory.provider().unwrap(); + let error = OverlayBuilder::::new( + blocks[3].recovered_block().hash(), + ChangesetCache::new(), + ) + .build_overlay(&provider) + .unwrap_err(); + + assert!( + error.to_string().contains("is after partial state trie frontier"), + "unexpected error: {error}" + ); + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn managed_overlay_errors_if_parent_is_not_persisted_or_managed_across_frontiers() { + let (factory, blocks) = setup_frontiers(1, 3); + let provider = factory.provider().unwrap(); + let parent_hash = blocks[3].recovered_block().hash(); + let error = OverlayBuilder::::new(parent_hash, ChangesetCache::new()) + .with_overlay_manager(OverlayManager::default()) + .build_overlay(&provider) + .unwrap_err(); + + assert!(matches!(error, ProviderError::BlockHashNotFound(hash) if hash == parent_hash)); + } + + #[test] + fn managed_overlay_skips_manager_for_persisted_parent() { + let parent_hash = B256::with_last_byte(1); + let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) + .with_overlay_manager(OverlayManager::default()); + + let (trie, state) = builder.resolve_overlays(parent_hash).unwrap(); + assert!(trie.is_empty()); + assert!(state.is_empty()); + } + + #[test] + fn managed_overlay_errors_if_parent_is_not_persisted_or_managed() { + let parent_hash = B256::with_last_byte(1); + let anchor_hash = B256::with_last_byte(2); + let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) + .with_overlay_manager(OverlayManager::default()); + + let err = builder.resolve_overlays(anchor_hash).unwrap_err(); + + assert!(err.to_string().contains("cannot be anchored")); + } + + #[test] + fn managed_overlay_skip_requires_both_frontiers() { + let parent_hash = B256::with_last_byte(1); + let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) + .with_overlay_manager(OverlayManager::default()); + assert!(!builder.should_skip_overlay_for_reused_sparse_trie(parent_hash, parent_hash)); + + let builder = builder.with_skip_overlay_for_reused_sparse_trie(parent_hash); + assert!(builder.should_skip_overlay_for_reused_sparse_trie(parent_hash, parent_hash)); + assert!(!builder + .should_skip_overlay_for_reused_sparse_trie(B256::with_last_byte(3), parent_hash,)); + + let blocks = test_blocks(); + let manager = OverlayManager::default(); + for block in &blocks[2..=4] { + manager.insert_block(block.clone()); + } + let builder = OverlayBuilder::::new( + blocks[4].recovered_block().hash(), + ChangesetCache::default(), + ) + .with_overlay_manager(manager) + .with_skip_overlay_for_reused_sparse_trie(blocks[1].recovered_block().hash()); + assert!(builder.should_skip_overlay_for_reused_sparse_trie( + blocks[1].recovered_block().hash(), + blocks[3].recovered_block().hash(), + )); + + let builder = + builder.with_skip_overlay_for_reused_sparse_trie(blocks[2].recovered_block().hash()); + assert!(!builder.should_skip_overlay_for_reused_sparse_trie( + blocks[1].recovered_block().hash(), + blocks[3].recovered_block().hash(), + )); + } +} diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs new file mode 100644 index 00000000000..ed062f06fb5 --- /dev/null +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -0,0 +1,1160 @@ +//! Trie changeset caching utilities. +//! +//! This module provides functionality to compute trie changesets for a given block, +//! which represent the old trie node values before the block was processed. +//! +//! It also provides an efficient in-memory cache for these changesets, which is essential for: +//! - **Reorg support**: Quickly access changesets to revert blocks during chain reorganizations +//! - **Memory efficiency**: Explicit eviction releases persisted changesets + +use alloy_primitives::{map::B256Map, BlockNumber, B256}; +use parking_lot::RwLock; +use reth_metrics::{ + metrics::{Counter, Gauge}, + Metrics, +}; +use reth_primitives_traits::FastInstant as Instant; +use reth_storage_api::{ + BlockNumReader, ChangeSetReader, DBProvider, StorageChangeSetReader, StorageSettingsCache, +}; +use reth_storage_errors::provider::{ProviderError, ProviderResult}; +use reth_trie::trie_cursor::{InMemoryTrieCursorFactory, TrieCursor, TrieCursorFactory}; +use reth_trie_common::updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}; +use reth_trie_db::{DatabaseTrieCursorFactory, TrieTableAdapter}; +use std::{ + collections::{BTreeMap, HashMap}, + ops::RangeInclusive, + sync::Arc, +}; +use tracing::{debug, warn}; + +#[cfg(test)] +use reth_trie::{changesets::compute_trie_changesets, TrieInputSorted}; +#[cfg(test)] +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; + +/// Computes block trie updates using the changeset cache. +/// +/// # Algorithm +/// +/// For block N: +/// 1. Get cumulative trie reverts from block N+1 to db tip using the cache +/// 2. Create an overlay cursor factory with these reverts (representing trie state after block N) +/// 3. Walk through account trie changesets for block N +/// 4. For each changed path, look up the current value using the overlay cursor +/// 5. Walk through storage trie changesets for block N +/// 6. For each changed path, look up the current value using the overlay cursor +/// 7. Return the collected trie updates +/// +/// # Arguments +/// +/// * `cache` - Handle to the changeset cache for retrieving trie reverts +/// * `provider` - Database provider for accessing changesets and block data +/// * `block_number` - Block number to compute trie updates for +/// +/// # Returns +/// +/// Trie updates representing the state of trie nodes after the block was processed +/// +/// # Errors +/// +/// Returns error if: +/// - Block number exceeds database tip +/// - Database access fails +/// - Cache retrieval fails +pub fn compute_block_trie_updates( + cache: &ChangesetCache, + provider: &Provider, + block_number: BlockNumber, +) -> ProviderResult +where + Provider: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, +{ + reth_trie_db::with_adapter!(provider, |A| { + compute_block_trie_updates_inner::<_, A>(cache, provider, block_number) + }) +} + +fn compute_block_trie_updates_inner( + cache: &ChangesetCache, + provider: &Provider, + block_number: BlockNumber, +) -> ProviderResult +where + Provider: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, + A: TrieTableAdapter, +{ + let tx = provider.tx_ref(); + + let db_tip_block = provider.best_block_number()?; + + // Step 1: Get the trie changesets for the target block from cache + let changesets = cache.get_or_compute(provider, block_number)?; + + // Step 2: Get the trie reverts for the state after the target block using the cache + let reverts = cache.get_or_compute_range(provider, (block_number + 1)..=db_tip_block)?; + + // Step 3: Create an InMemoryTrieCursorFactory with the reverts + // This gives us the trie state as it was after the target block was processed + let db_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx); + let cursor_factory = InMemoryTrieCursorFactory::new(db_cursor_factory, &reverts); + + // Step 4: Collect all account trie nodes that changed in the target block + let account_nodes_ref = changesets.account_nodes_ref(); + let mut account_nodes = Vec::with_capacity(account_nodes_ref.len()); + let mut account_cursor = cursor_factory.account_trie_cursor()?; + + // Iterate over the account nodes from the changesets + for (nibbles, _old_node) in account_nodes_ref { + // Look up the current value of this trie node using the overlay cursor + let node_value = account_cursor.seek_exact(*nibbles)?.map(|(_, node)| node); + account_nodes.push((*nibbles, node_value)); + } + + // Step 5: Collect all storage trie nodes that changed in the target block + let mut storage_tries = B256Map::default(); + + // Iterate over the storage tries from the changesets + for (hashed_address, storage_changeset) in changesets.storage_tries_ref() { + let mut storage_cursor = cursor_factory.storage_trie_cursor(*hashed_address)?; + let storage_nodes_ref = storage_changeset.storage_nodes_ref(); + let mut storage_nodes = Vec::with_capacity(storage_nodes_ref.len()); + + // Iterate over the storage nodes for this account + for (nibbles, _old_node) in storage_nodes_ref { + // Look up the current value of this storage trie node + let node_value = storage_cursor.seek_exact(*nibbles)?.map(|(_, node)| node); + storage_nodes.push((*nibbles, node_value)); + } + + storage_tries.insert( + *hashed_address, + StorageTrieUpdatesSorted { storage_nodes, is_deleted: storage_changeset.is_deleted }, + ); + } + + Ok(TrieUpdatesSorted::new(account_nodes, storage_tries)) +} + +/// Thread-safe changeset cache. +/// +/// This type wraps a shared, mutable reference to the cache inner. +/// The `RwLock` enables concurrent reads while ensuring exclusive access for writes. +#[derive(Debug, Clone)] +pub struct ChangesetCache { + inner: Arc>, +} + +impl Default for ChangesetCache { + fn default() -> Self { + Self::new() + } +} + +impl ChangesetCache { + /// Creates a new cache. + /// + /// The cache has no capacity limit and relies on explicit eviction + /// via the `evict()` method to manage memory usage. + pub fn new() -> Self { + Self { inner: Arc::new(RwLock::new(ChangesetCacheInner::new())) } + } + + /// Retrieves changesets for a block. + /// + /// Returns `None` if the block is not in the cache (either evicted or never computed). + /// Updates hit/miss metrics accordingly. + pub fn get( + &self, + block_hash: B256, + block_number: BlockNumber, + ) -> Option> { + self.inner.read().get(&ChangesetRangeKey::single(block_number, block_hash)) + } + + /// Evicts changesets for blocks below the given block number. + /// + /// This should be called after blocks are persisted to the database to free + /// memory for changesets that are no longer needed in the cache. + /// + /// # Arguments + /// + /// * `up_to_block` - Evict blocks with number < this value. Blocks with number >= this value + /// are retained. + pub fn evict(&self, up_to_block: BlockNumber) { + self.inner.write().evict(up_to_block) + } + + /// Gets changesets from cache, or computes them on-the-fly if missing. + /// + /// This is the primary API for retrieving changesets. It checks the cache first, then falls + /// back to computing from database state if missing. + /// + /// # Arguments + /// + /// * `block_number` - Block number (for cache insertion and logging) + /// * `provider` - Database provider for DB access + /// + /// # Returns + /// + /// Changesets for the block, either from cache or computed on-the-fly. + pub fn get_or_compute

( + &self, + provider: &P, + block_number: BlockNumber, + ) -> ProviderResult> + where + P: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, + { + self.get_or_compute_range(provider, block_number..=block_number) + } + + /// Gets or computes trie reverts for a range of blocks. + /// + /// If all blocks in the range are cached, this method retrieves and accumulates those + /// per-block trie changesets (reverts) in reverse order (newest to oldest), so that older + /// values take precedence when there are conflicts. + /// + /// If any block is missing from cache, this falls back to one aggregate database computation + /// for the whole range. The aggregate result restores the trie to the state before the range + /// and is inserted into the range cache. + /// + /// # Arguments + /// + /// * `provider` - Database provider for DB access and block lookups + /// * `range` - Block range to accumulate reverts for (inclusive) + /// + /// # Returns + /// + /// Accumulated trie reverts for all blocks in the specified range + /// + /// # Errors + /// + /// Returns error if: + /// - Any block in the range is beyond the database tip + /// - Database access fails + /// - Block hash lookup fails + /// - Changeset computation fails + pub fn get_or_compute_range

( + &self, + provider: &P, + range: RangeInclusive, + ) -> ProviderResult> + where + P: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, + { + let db_tip_block = provider.best_block_number()?; + + let start_block = *range.start(); + let end_block = *range.end(); + + // If range end is beyond the tip, return an error + if end_block > db_tip_block { + return Err(ProviderError::InsufficientChangesets { + requested: end_block, + available: 0..=db_tip_block, + }); + } + + let timer = Instant::now(); + + debug!( + target: "trie::changeset_cache", + start_block, + end_block, + db_tip_block, + "Starting get_or_compute_range" + ); + + if start_block > end_block { + debug!( + target: "trie::changeset_cache", + start_block, + end_block, + "Empty changeset range requested" + ); + return Ok(Arc::new(TrieUpdatesSorted::default())) + } + + let end_block_hash = provider.block_hash(end_block)?.ok_or_else(|| { + ProviderError::other(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("block hash not found for block number {}", end_block), + )) + })?; + let range_key = ChangesetRangeKey::new(start_block, end_block, end_block_hash); + + if let Some(accumulated_reverts) = self.inner.read().get(&range_key) { + let elapsed = timer.elapsed(); + + debug!( + target: "trie::changeset_cache", + ?elapsed, + start_block, + end_block, + ?end_block_hash, + num_blocks = end_block.saturating_sub(start_block).saturating_add(1), + "Changeset cache HIT for block range" + ); + + return Ok(accumulated_reverts) + } + + let mut cached_reverts = + Vec::with_capacity(end_block.saturating_sub(start_block).saturating_add(1) as usize); + let mut all_cached = true; + + for block_number in range.rev() { + // Get the block hash for this block number + let block_hash = if block_number == end_block { + end_block_hash + } else { + provider.block_hash(block_number)?.ok_or_else(|| { + ProviderError::other(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("block hash not found for block number {}", block_number), + )) + })? + }; + + debug!( + target: "trie::changeset_cache", + block_number, + ?block_hash, + "Looked up block hash for block number in range" + ); + + let block_key = ChangesetRangeKey::single(block_number, block_hash); + if let Some(changesets) = self.inner.read().get(&block_key) { + cached_reverts.push(changesets); + } else { + all_cached = false; + break + } + } + + if all_cached { + // `merge_slice` gives precedence to earlier items, so pass reverts oldest-to-newest. + cached_reverts.reverse(); + let accumulated_reverts = Arc::new(TrieUpdatesSorted::merge_slice(&cached_reverts)); + let elapsed = timer.elapsed(); + + let num_account_nodes = accumulated_reverts.account_nodes_ref().len(); + let num_storage_tries = accumulated_reverts.storage_tries_ref().len(); + + debug!( + target: "trie::changeset_cache", + ?elapsed, + start_block, + end_block, + num_blocks = end_block.saturating_sub(start_block).saturating_add(1), + num_account_nodes, + num_storage_tries, + "Finished accumulating cached trie reverts for block range" + ); + + self.inner.write().insert(range_key, Arc::clone(&accumulated_reverts)); + return Ok(accumulated_reverts) + } + + warn!( + target: "trie::changeset_cache", + start_block, + end_block, + "Changeset cache MISS in range, falling back to aggregate DB-based computation" + ); + + let accumulated_reverts = Arc::new(reth_trie_db::compute_range_trie_changesets( + provider, + start_block..=end_block, + db_tip_block, + )?); + + let elapsed = timer.elapsed(); + + let num_account_nodes = accumulated_reverts.account_nodes_ref().len(); + let num_storage_tries = accumulated_reverts.storage_tries_ref().len(); + + debug!( + target: "trie::changeset_cache", + ?elapsed, + start_block, + end_block, + ?end_block_hash, + num_blocks = end_block.saturating_sub(start_block).saturating_add(1), + num_account_nodes, + num_storage_tries, + "Finished accumulating trie reverts for block range" + ); + + self.inner.write().insert(range_key, Arc::clone(&accumulated_reverts)); + + Ok(accumulated_reverts) + } +} + +/// Cache key for one contiguous range of canonical trie changesets. +/// +/// The end block hash disambiguates canonical rewrites where the same block numbers later refer to +/// a different chain. For a single block, `start_block == end_block` and `end_block_hash` is that +/// block's hash. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +struct ChangesetRangeKey { + start_block: BlockNumber, + end_block: BlockNumber, + end_block_hash: B256, +} + +impl ChangesetRangeKey { + const fn new(start_block: BlockNumber, end_block: BlockNumber, end_block_hash: B256) -> Self { + Self { start_block, end_block, end_block_hash } + } + + const fn single(block_number: BlockNumber, block_hash: B256) -> Self { + Self::new(block_number, block_number, block_hash) + } +} + +/// In-memory cache for trie changesets with explicit eviction policy. +/// +/// Holds changesets for blocks or block ranges that have been validated but not yet persisted. +/// Keyed by canonical block range. Eviction is controlled +/// explicitly by the engine API tree handler when persistence completes. +/// +/// ## Eviction Policy +/// +/// Unlike traditional caches with automatic eviction, this cache requires explicit +/// eviction calls. The engine API tree handler calls `evict(block_number)` after +/// blocks are persisted to the database, ensuring changesets remain available +/// until their corresponding blocks are safely on disk. +/// +/// ## Metrics +/// +/// The cache maintains several metrics for observability: +/// - `hits`: Number of successful cache lookups +/// - `misses`: Number of failed cache lookups +/// - `evictions`: Number of blocks evicted +/// - `size`: Current number of cached blocks +#[derive(Debug)] +struct ChangesetCacheInner { + /// Cache entries keyed by inclusive block range plus the range's canonical end hash. + entries: HashMap>, + + /// Range start block to cache keys mapping for eviction. + range_starts: BTreeMap>, + + /// Metrics for monitoring cache behavior + metrics: ChangesetCacheMetrics, +} + +/// Metrics for the changeset cache. +/// +/// These metrics provide visibility into cache performance and help identify +/// potential issues like high miss rates. +#[derive(Metrics, Clone)] +#[metrics(scope = "trie.changeset_cache")] +struct ChangesetCacheMetrics { + /// Cache hit counter + hits: Counter, + + /// Cache miss counter + misses: Counter, + + /// Eviction counter + evictions: Counter, + + /// Current cache size (number of entries) + size: Gauge, +} + +impl Default for ChangesetCacheInner { + fn default() -> Self { + Self::new() + } +} + +impl ChangesetCacheInner { + /// Creates a new empty changeset cache. + /// + /// The cache has no capacity limit and relies on explicit eviction + /// via the `evict()` method to manage memory usage. + fn new() -> Self { + Self { entries: HashMap::new(), range_starts: BTreeMap::new(), metrics: Default::default() } + } + + fn get(&self, key: &ChangesetRangeKey) -> Option> { + match self.entries.get(key) { + Some(changesets) => { + self.metrics.hits.increment(1); + Some(Arc::clone(changesets)) + } + None => { + self.metrics.misses.increment(1); + None + } + } + } + + fn insert(&mut self, key: ChangesetRangeKey, changesets: Arc) { + debug!( + target: "trie::changeset_cache", + ?key, + cache_size_before = self.entries.len(), + "Inserting changeset into cache" + ); + + let is_new_entry = self.entries.insert(key, changesets).is_none(); + + if is_new_entry { + self.range_starts.entry(key.start_block).or_default().push(key); + } + + // Update size metric + self.metrics.size.set(self.entries.len() as f64); + + debug!( + target: "trie::changeset_cache", + ?key, + cache_size_after = self.entries.len(), + "Changeset inserted into cache" + ); + } + + fn evict(&mut self, up_to_block: BlockNumber) { + debug!( + target: "trie::changeset_cache", + up_to_block, + cache_size_before = self.entries.len(), + "Starting cache eviction" + ); + + // Find all block numbers that should be evicted (< up_to_block) + let range_starts_to_evict: Vec = + self.range_starts.range(..up_to_block).map(|(num, _)| *num).collect(); + + // Remove entries for each block number below threshold + let mut evicted_count = 0; + + for start_block in &range_starts_to_evict { + if let Some(keys) = self.range_starts.remove(start_block) { + debug!( + target: "trie::changeset_cache", + start_block, + num_ranges = keys.len(), + "Evicting ranges from cache" + ); + for key in keys { + if self.entries.remove(&key).is_some() { + evicted_count += 1; + } + } + } + } + + debug!( + target: "trie::changeset_cache", + up_to_block, + evicted_count, + cache_size_after = self.entries.len(), + "Finished cache eviction" + ); + + // Update metrics if we evicted anything + if evicted_count > 0 { + self.metrics.evictions.increment(evicted_count as u64); + self.metrics.size.set(self.entries.len() as f64); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::Header; + use alloy_primitives::{ + keccak256, + map::{B256Map, HashMap}, + Address, U256, + }; + use reth_db::{ + models::{AccountBeforeTx, BlockNumberAddress}, + tables, + transaction::DbTxMut, + }; + use reth_primitives_traits::{Account, StorageEntry}; + use reth_provider::{ + test_utils::create_test_provider_factory, StaticFileProviderFactory, StaticFileSegment, + StaticFileWriter, + }; + use reth_stages_types::{StageCheckpoint, StageId}; + use reth_storage_api::{StageCheckpointWriter, TrieWriter}; + use reth_trie::{BranchNodeCompact, Nibbles, StateRoot}; + + // Helper function to create empty TrieUpdatesSorted for testing + fn create_test_changesets() -> Arc { + Arc::new(TrieUpdatesSorted::new(vec![], B256Map::default())) + } + + fn insert_test_changesets( + cache: &mut ChangesetCacheInner, + block_hash: B256, + block_number: BlockNumber, + changesets: Arc, + ) { + cache.insert(ChangesetRangeKey::single(block_number, block_hash), changesets); + } + + fn get_test_changesets( + cache: &ChangesetCacheInner, + block_hash: B256, + block_number: BlockNumber, + ) -> Option> { + cache.get(&ChangesetRangeKey::single(block_number, block_hash)) + } + + fn test_account(balance: u64) -> Account { + Account { balance: U256::from(balance), ..Default::default() } + } + + fn test_storage(slot: u64, value: u64) -> StorageEntry { + StorageEntry { key: B256::from(U256::from(slot)), value: U256::from(value) } + } + + fn seed_headers( + factory: &impl StaticFileProviderFactory< + Primitives: reth_primitives_traits::NodePrimitives, + >, + end_block: BlockNumber, + ) { + let static_file_provider = factory.static_file_provider(); + let mut header_writer = + static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + for block_number in 0..=end_block { + let header = Header { number: block_number, ..Default::default() }; + header_writer + .append_header(&header, &B256::with_last_byte(block_number as u8)) + .unwrap(); + } + header_writer.commit().unwrap(); + } + + fn legacy_compute_range_trie_changesets( + provider: &Provider, + range: RangeInclusive, + ) -> TrieUpdatesSorted + where + Provider: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, + { + let mut accumulated_reverts = TrieUpdatesSorted::default(); + for block_number in range.rev() { + let changesets = legacy_compute_block_trie_changesets(provider, block_number); + accumulated_reverts.extend_ref_and_sort(&changesets); + } + accumulated_reverts + } + + fn legacy_compute_block_trie_changesets( + provider: &Provider, + block_number: BlockNumber, + ) -> TrieUpdatesSorted + where + Provider: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, + { + reth_trie_db::with_adapter!(provider, |A| { + legacy_compute_block_trie_changesets_inner::<_, A>(provider, block_number) + }) + } + + fn legacy_compute_block_trie_changesets_inner( + provider: &Provider, + block_number: BlockNumber, + ) -> TrieUpdatesSorted + where + Provider: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, + A: TrieTableAdapter, + { + let individual_state_revert = + reth_trie_db::from_reverts_auto(provider, block_number..=block_number).unwrap(); + let cumulative_state_revert = + reth_trie_db::from_reverts_auto(provider, (block_number + 1)..).unwrap(); + + let mut cumulative_state_revert_prev = cumulative_state_revert.clone(); + cumulative_state_revert_prev.extend_ref_and_sort(&individual_state_revert); + + type DbStateRoot<'a, TX, A> = + StateRoot, DatabaseHashedCursorFactory<&'a TX>>; + + let input_prev = TrieInputSorted::new( + Arc::default(), + Arc::new(cumulative_state_revert_prev.clone()), + cumulative_state_revert_prev.construct_prefix_sets(), + ); + let cumulative_trie_updates_prev = + DbStateRoot::<_, A>::overlay_root_from_nodes_with_updates( + provider.tx_ref(), + input_prev, + ) + .unwrap() + .1 + .into_sorted(); + + let input = TrieInputSorted::new( + Arc::new(cumulative_trie_updates_prev.clone()), + Arc::new(cumulative_state_revert), + individual_state_revert.construct_prefix_sets(), + ); + let trie_updates = + DbStateRoot::<_, A>::overlay_root_from_nodes_with_updates(provider.tx_ref(), input) + .unwrap() + .1 + .into_sorted(); + + let db_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(provider.tx_ref()); + let overlay_factory = + InMemoryTrieCursorFactory::new(db_cursor_factory, &cumulative_trie_updates_prev); + + compute_trie_changesets(&overlay_factory, &trie_updates).unwrap() + } + + fn seed_tip_trie_tables(provider: &Provider) + where + Provider: DBProvider + TrieWriter, + A: TrieTableAdapter, + { + type DbStateRoot<'a, TX, A> = + StateRoot, DatabaseHashedCursorFactory<&'a TX>>; + + let (_, trie_updates) = + DbStateRoot::<_, A>::from_tx(provider.tx_ref()).root_with_updates().unwrap(); + provider.write_trie_updates(trie_updates).unwrap(); + } + + #[test] + fn cached_range_merge_keeps_oldest_revert_values() { + let factory = create_test_provider_factory(); + seed_headers(&factory, 2); + + let provider = factory.provider_rw().unwrap(); + provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(2)).unwrap(); + + let cache = ChangesetCache::new(); + let path = Nibbles::from_nibbles([0x1, 0x2]); + let older_node = BranchNodeCompact::new(0b0001, 0, 0, vec![], None); + let newer_node = BranchNodeCompact::new(0b0010, 0, 0, vec![], None); + + { + let mut cache = cache.inner.write(); + insert_test_changesets( + &mut cache, + B256::with_last_byte(1), + 1, + Arc::new(TrieUpdatesSorted::new( + vec![(path, Some(older_node.clone()))], + B256Map::default(), + )), + ); + insert_test_changesets( + &mut cache, + B256::with_last_byte(2), + 2, + Arc::new(TrieUpdatesSorted::new( + vec![(path, Some(newer_node))], + B256Map::default(), + )), + ); + } + + let accumulated = cache.get_or_compute_range(&*provider, 1..=2).unwrap(); + assert_eq!(accumulated.account_nodes_ref(), &[(path, Some(older_node))]); + } + + #[test] + fn aggregate_range_reverts_to_pre_range_state() { + let factory = create_test_provider_factory(); + seed_headers(&factory, 3); + + let provider = factory.provider_rw().unwrap(); + let address = Address::with_last_byte(1); + let hashed_address = keccak256(address); + let slot1 = B256::from(U256::from(1)); + let slot2 = B256::from(U256::from(2)); + let account1 = test_account(10); + let account2 = test_account(20); + let account3 = test_account(30); + + provider.tx_ref().put::(hashed_address, account3).unwrap(); + provider + .tx_ref() + .put::( + hashed_address, + StorageEntry { key: keccak256(slot1), value: U256::from(25) }, + ) + .unwrap(); + provider + .tx_ref() + .put::( + hashed_address, + StorageEntry { key: keccak256(slot2), value: U256::from(20) }, + ) + .unwrap(); + + provider + .tx_ref() + .put::(1, AccountBeforeTx { address, info: None }) + .unwrap(); + provider + .tx_ref() + .put::(2, AccountBeforeTx { address, info: Some(account1) }) + .unwrap(); + provider + .tx_ref() + .put::(3, AccountBeforeTx { address, info: Some(account2) }) + .unwrap(); + + provider + .tx_ref() + .put::(BlockNumberAddress((1, address)), test_storage(1, 0)) + .unwrap(); + provider + .tx_ref() + .put::(BlockNumberAddress((1, address)), test_storage(2, 0)) + .unwrap(); + provider + .tx_ref() + .put::( + BlockNumberAddress((2, address)), + StorageEntry { key: slot1, value: U256::from(10) }, + ) + .unwrap(); + provider + .tx_ref() + .put::( + BlockNumberAddress((3, address)), + StorageEntry { key: slot1, value: U256::from(15) }, + ) + .unwrap(); + + provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(3)).unwrap(); + reth_trie_db::with_adapter!(provider, |A| seed_tip_trie_tables::<_, A>(&*provider)); + + let actual = reth_trie_db::compute_range_trie_changesets(&*provider, 1..=3, 3).unwrap(); + let storage_revert = actual + .storage_tries_ref() + .get(&hashed_address) + .expect("created account storage trie should be deleted by range revert"); + assert!(storage_revert.is_deleted()); + assert!(storage_revert.storage_nodes_ref().is_empty()); + + let cache = ChangesetCache::new(); + let from_cache_api = cache.get_or_compute_range(&*provider, 1..=3).unwrap(); + assert_eq!(*from_cache_api, actual); + assert_eq!(cache.inner.read().entries.len(), 1); + + let block_changesets = cache.get_or_compute(&*provider, 2).unwrap(); + assert_eq!(*block_changesets, legacy_compute_block_trie_changesets(&*provider, 2)); + assert_eq!(cache.inner.read().entries.len(), 2); + } + + #[test] + fn aggregate_range_matches_legacy_per_block_merge_with_storage_wipe() { + let factory = create_test_provider_factory(); + seed_headers(&factory, 3); + + let provider = factory.provider_rw().unwrap(); + let address = Address::with_last_byte(1); + let slot1 = B256::from(U256::from(1)); + let slot2 = B256::from(U256::from(2)); + let account1 = test_account(10); + let account2 = test_account(20); + + provider + .tx_ref() + .put::(1, AccountBeforeTx { address, info: None }) + .unwrap(); + provider + .tx_ref() + .put::(2, AccountBeforeTx { address, info: Some(account1) }) + .unwrap(); + provider + .tx_ref() + .put::(3, AccountBeforeTx { address, info: Some(account2) }) + .unwrap(); + + provider + .tx_ref() + .put::(BlockNumberAddress((1, address)), test_storage(1, 0)) + .unwrap(); + provider + .tx_ref() + .put::(BlockNumberAddress((1, address)), test_storage(2, 0)) + .unwrap(); + provider + .tx_ref() + .put::( + BlockNumberAddress((2, address)), + StorageEntry { key: slot1, value: U256::from(10) }, + ) + .unwrap(); + provider + .tx_ref() + .put::( + BlockNumberAddress((3, address)), + StorageEntry { key: slot1, value: U256::from(15) }, + ) + .unwrap(); + provider + .tx_ref() + .put::( + BlockNumberAddress((3, address)), + StorageEntry { key: slot2, value: U256::from(20) }, + ) + .unwrap(); + + provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(3)).unwrap(); + reth_trie_db::with_adapter!(provider, |A| seed_tip_trie_tables::<_, A>(&*provider)); + + let expected = legacy_compute_range_trie_changesets(&*provider, 2..=3); + let actual = reth_trie_db::compute_range_trie_changesets(&*provider, 2..=3, 3).unwrap(); + assert_eq!(actual, expected); + } + + #[test] + fn test_insert_and_retrieve_single_entry() { + let mut cache = ChangesetCacheInner::new(); + let hash = B256::random(); + let changesets = create_test_changesets(); + + insert_test_changesets(&mut cache, hash, 100, Arc::clone(&changesets)); + + // Should be able to retrieve it + let retrieved = get_test_changesets(&cache, hash, 100); + assert!(retrieved.is_some()); + assert_eq!(cache.entries.len(), 1); + } + + #[test] + fn test_insert_multiple_entries() { + let mut cache = ChangesetCacheInner::new(); + + // Insert 10 blocks + let mut hashes = Vec::new(); + for i in 0..10 { + let hash = B256::random(); + insert_test_changesets(&mut cache, hash, 100 + i, create_test_changesets()); + hashes.push((100 + i, hash)); + } + + // Should be able to retrieve all + assert_eq!(cache.entries.len(), 10); + for (block_number, hash) in hashes { + assert!(get_test_changesets(&cache, hash, block_number).is_some()); + } + } + + #[test] + fn test_eviction_when_explicitly_called() { + let mut cache = ChangesetCacheInner::new(); + + // Insert 15 blocks (0-14) + let mut hashes = Vec::new(); + for i in 0..15 { + let hash = B256::random(); + insert_test_changesets(&mut cache, hash, i, create_test_changesets()); + hashes.push((i, hash)); + } + + // All blocks should be present (no automatic eviction) + assert_eq!(cache.entries.len(), 15); + + // Explicitly evict blocks < 4 + cache.evict(4); + + // Blocks 0-3 should be evicted + assert_eq!(cache.entries.len(), 11); // blocks 4-14 = 11 blocks + + // Verify blocks 0-3 are evicted + for i in 0..4 { + assert!( + get_test_changesets(&cache, hashes[i as usize].1, i).is_none(), + "Block {} should be evicted", + i + ); + } + + // Verify blocks 4-14 are still present + for i in 4..15 { + assert!( + get_test_changesets(&cache, hashes[i as usize].1, i).is_some(), + "Block {} should be present", + i + ); + } + } + + #[test] + fn test_eviction_with_persistence_watermark() { + let mut cache = ChangesetCacheInner::new(); + + // Insert blocks 100-165 + let mut hashes = HashMap::new(); + for i in 100..=165 { + let hash = B256::random(); + insert_test_changesets(&mut cache, hash, i, create_test_changesets()); + hashes.insert(i, hash); + } + + // All blocks should be present (no automatic eviction) + assert_eq!(cache.entries.len(), 66); + + // Simulate persistence up to block 164, with 64-block retention window + // Eviction threshold = 164 - 64 = 100 + cache.evict(100); + + // Blocks 100-165 should remain (66 blocks) + assert_eq!(cache.entries.len(), 66); + + // Simulate persistence up to block 165 + // Eviction threshold = 165 - 64 = 101 + cache.evict(101); + + // Blocks 101-165 should remain (65 blocks) + assert_eq!(cache.entries.len(), 65); + assert!(get_test_changesets(&cache, hashes[&100], 100).is_none()); + assert!(get_test_changesets(&cache, hashes[&101], 101).is_some()); + } + + #[test] + fn test_out_of_order_inserts_with_explicit_eviction() { + let mut cache = ChangesetCacheInner::new(); + + // Insert blocks in random order + let hash_10 = B256::random(); + insert_test_changesets(&mut cache, hash_10, 10, create_test_changesets()); + + let hash_5 = B256::random(); + insert_test_changesets(&mut cache, hash_5, 5, create_test_changesets()); + + let hash_15 = B256::random(); + insert_test_changesets(&mut cache, hash_15, 15, create_test_changesets()); + + let hash_3 = B256::random(); + insert_test_changesets(&mut cache, hash_3, 3, create_test_changesets()); + + // All blocks should be present (no automatic eviction) + assert_eq!(cache.entries.len(), 4); + + // Explicitly evict blocks < 5 + cache.evict(5); + + assert!(get_test_changesets(&cache, hash_3, 3).is_none(), "Block 3 should be evicted"); + assert!(get_test_changesets(&cache, hash_5, 5).is_some(), "Block 5 should be present"); + assert!(get_test_changesets(&cache, hash_10, 10).is_some(), "Block 10 should be present"); + assert!(get_test_changesets(&cache, hash_15, 15).is_some(), "Block 15 should be present"); + } + + #[test] + fn test_multiple_blocks_same_number() { + let mut cache = ChangesetCacheInner::new(); + + // Insert multiple blocks with same number (side chains) + let hash_1a = B256::random(); + let hash_1b = B256::random(); + insert_test_changesets(&mut cache, hash_1a, 100, create_test_changesets()); + insert_test_changesets(&mut cache, hash_1b, 100, create_test_changesets()); + + // Both should be retrievable + assert!(get_test_changesets(&cache, hash_1a, 100).is_some()); + assert!(get_test_changesets(&cache, hash_1b, 100).is_some()); + assert_eq!(cache.entries.len(), 2); + } + + #[test] + fn test_ranges_with_same_numbers_and_different_end_hashes_are_distinct() { + let mut cache = ChangesetCacheInner::new(); + let path = Nibbles::from_nibbles_unchecked([0x01]); + let hash_a = B256::with_last_byte(1); + let hash_b = B256::with_last_byte(2); + let key_a = ChangesetRangeKey::new(10, 20, hash_a); + let key_b = ChangesetRangeKey::new(10, 20, hash_b); + let changesets_a = Arc::new(TrieUpdatesSorted::new( + vec![(path, Some(BranchNodeCompact::new(0b0001, 0, 0, vec![], None)))], + B256Map::default(), + )); + let changesets_b = Arc::new(TrieUpdatesSorted::new( + vec![(path, Some(BranchNodeCompact::new(0b0010, 0, 0, vec![], None)))], + B256Map::default(), + )); + + cache.insert(key_a, Arc::clone(&changesets_a)); + cache.insert(key_b, Arc::clone(&changesets_b)); + + assert_eq!(cache.entries.len(), 2); + assert_eq!( + cache.get(&key_a).unwrap().account_nodes_ref(), + changesets_a.account_nodes_ref() + ); + assert_eq!( + cache.get(&key_b).unwrap().account_nodes_ref(), + changesets_b.account_nodes_ref() + ); + + cache.evict(11); + assert!(cache.get(&key_a).is_none()); + assert!(cache.get(&key_b).is_none()); + } + + #[test] + fn test_eviction_removes_all_side_chains() { + let mut cache = ChangesetCacheInner::new(); + + // Insert multiple blocks at same height (side chains) + let hash_10a = B256::random(); + let hash_10b = B256::random(); + let hash_10c = B256::random(); + insert_test_changesets(&mut cache, hash_10a, 10, create_test_changesets()); + insert_test_changesets(&mut cache, hash_10b, 10, create_test_changesets()); + insert_test_changesets(&mut cache, hash_10c, 10, create_test_changesets()); + + let hash_20 = B256::random(); + insert_test_changesets(&mut cache, hash_20, 20, create_test_changesets()); + + assert_eq!(cache.entries.len(), 4); + + // Evict blocks < 15 - should remove all three side chains at height 10 + cache.evict(15); + + assert_eq!(cache.entries.len(), 1); + assert!(get_test_changesets(&cache, hash_10a, 10).is_none()); + assert!(get_test_changesets(&cache, hash_10b, 10).is_none()); + assert!(get_test_changesets(&cache, hash_10c, 10).is_none()); + assert!(get_test_changesets(&cache, hash_20, 20).is_some()); + } +} diff --git a/crates/storage/storage-overlay/src/lib.rs b/crates/storage/storage-overlay/src/lib.rs new file mode 100644 index 00000000000..dac37519f25 --- /dev/null +++ b/crates/storage/storage-overlay/src/lib.rs @@ -0,0 +1,12 @@ +//! State trie overlay construction and caching. + +#![cfg_attr(not(test), warn(unused_crate_dependencies))] + +mod builder; +pub use builder::*; + +mod changeset_cache; +pub use changeset_cache::*; + +mod manager; +pub use manager::*; diff --git a/crates/chain-state/src/state_trie_overlay.rs b/crates/storage/storage-overlay/src/manager.rs similarity index 93% rename from crates/chain-state/src/state_trie_overlay.rs rename to crates/storage/storage-overlay/src/manager.rs index a56e1f2a693..21306ef9d73 100644 --- a/crates/chain-state/src/state_trie_overlay.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -1,12 +1,13 @@ //! Flattened state trie overlays for in-memory blocks. //! //! Payload validation needs a view of the state trie as of an in-memory parent block even when that -//! parent has not been persisted yet. [`StateTrieOverlayManager`] tracks those in-memory blocks and +//! parent has not been persisted yet. [`OverlayManager`] tracks those in-memory blocks and //! builds reusable flattened state trie overlays on demand. -use crate::{EthPrimitives, ExecutedBlock, PreservedSparseTrie}; use alloy_primitives::B256; use parking_lot::Mutex; +use reth_chain_state::{ExecutedBlock, PreservedSparseTrie}; +use reth_ethereum_primitives::EthPrimitives; use reth_metrics::{ metrics::{Counter, Histogram}, Metrics, @@ -30,7 +31,7 @@ use tracing::{debug, trace}; /// The manager owns the in-memory block graph and a cache of flattened state trie overlays keyed by /// `(anchor_hash, tip_hash)`. #[derive(Clone)] -pub struct StateTrieOverlayManager { +pub struct OverlayManager { blocks: Arc>>, overlays: Arc>, preserved_sparse_trie: Arc>>, @@ -51,7 +52,7 @@ struct StateTrieOverlayMetrics { overlay_cache_fills: Counter, } -impl Default for StateTrieOverlayManager { +impl Default for OverlayManager { fn default() -> Self { Self { blocks: Default::default(), @@ -64,17 +65,17 @@ impl Default for StateTrieOverlayManager { } } -impl std::fmt::Debug for StateTrieOverlayManager { +impl std::fmt::Debug for OverlayManager { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("StateTrieOverlayManager") + f.debug_struct("OverlayManager") .field("blocks", &self.blocks.len()) .field("overlays", &self.overlays.len()) .finish() } } -impl StateTrieOverlayManager { - /// Create a new [`StateTrieOverlayManager`] backed by the given worker pool. +impl OverlayManager { + /// Create a new [`OverlayManager`] backed by the given worker pool. #[cfg(feature = "rayon")] pub fn new(worker_pool: Arc) -> Self { Self { @@ -111,7 +112,7 @@ impl StateTrieOverlayManager { let elapsed = start.elapsed(); if elapsed.as_millis() > 5 { debug!( - target: "engine::tree::payload_processor", + target: "storage::overlay::manager", blocked_for=?elapsed, "Waited for preserved sparse trie to become available" ); @@ -122,7 +123,7 @@ impl StateTrieOverlayManager { /// Inserts an executed in-memory block into the state trie overlay manager. #[tracing::instrument( level = "trace", - target = "chain_state::state_trie_overlay", + target = "storage::overlay::manager", skip_all, fields( block_hash = %block.recovered_block().hash(), @@ -140,7 +141,7 @@ impl StateTrieOverlayManager { Entry::Occupied(_) => { span.record("duplicate", true); debug!( - target: "chain_state::state_trie_overlay", + target: "storage::overlay::manager", %hash, %parent_hash, "state trie overlay block already inserted" @@ -153,7 +154,7 @@ impl StateTrieOverlayManager { } debug!( - target: "chain_state::state_trie_overlay", + target: "storage::overlay::manager", %hash, %parent_hash, "inserted block into state trie overlay manager" @@ -164,7 +165,7 @@ impl StateTrieOverlayManager { /// built from the remaining blocks. #[tracing::instrument( level = "trace", - target = "chain_state::state_trie_overlay", + target = "storage::overlay::manager", skip_all, fields( block_count = tracing::field::Empty, @@ -198,7 +199,7 @@ impl StateTrieOverlayManager { span.record("pruned_overlays", pruned_overlays); } debug!( - target: "chain_state::state_trie_overlay", + target: "storage::overlay::manager", block_count, removed_blocks, pruned_overlays, @@ -209,7 +210,7 @@ impl StateTrieOverlayManager { /// Returns the flattened overlay from `anchor_hash` to `parent_hash`. #[tracing::instrument( level = "trace", - target = "chain_state::state_trie_overlay", + target = "storage::overlay::manager", skip_all, fields(tip_hash = %parent_hash, anchor_hash = %anchor_hash) )] @@ -219,7 +220,7 @@ impl StateTrieOverlayManager { anchor_hash: B256, ) -> Result<(Arc, Arc), StateTrieOverlayError> { debug!( - target: "chain_state::state_trie_overlay", + target: "storage::overlay::manager", tip_hash = %parent_hash, %anchor_hash, "loading state trie overlay for parent" @@ -230,7 +231,7 @@ impl StateTrieOverlayManager { #[tracing::instrument( level = "trace", - target = "chain_state::state_trie_overlay", + target = "storage::overlay::manager", skip_all, fields( tip_hash = %tip_hash, @@ -480,7 +481,7 @@ enum ComputeOverlayInput { #[tracing::instrument( level = "trace", - target = "chain_state::state_trie_overlay", + target = "storage::overlay::manager", skip_all, fields( anchor_hash = %anchor_hash, @@ -508,7 +509,7 @@ fn compute_overlay( let trie_data = block.trie_data(); trace!( - target: "chain_state::state_trie_overlay", + target: "storage::overlay::manager", %anchor_hash, head = %block.recovered_block().hash(), "extending cached parent state trie overlay" @@ -529,7 +530,7 @@ fn compute_overlay( metrics.overlay_computation_duration_seconds.record(elapsed.as_secs_f64()); tracing::Span::current().record("elapsed_us", elapsed.as_micros() as u64); debug!( - target: "chain_state::state_trie_overlay", + target: "storage::overlay::manager", %anchor_hash, block_count, parent_overlay, @@ -605,8 +606,9 @@ fn extend_overlay( #[cfg(test)] mod tests { use super::*; - use crate::{test_utils::TestBlockBuilder, EthPrimitives, ExecutedBlock, SparseTrie}; use alloy_primitives::U256; + use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock, SparseTrie}; + use reth_ethereum_primitives::EthPrimitives; use reth_primitives_traits::Account; use reth_trie::{updates::TrieUpdatesSorted, ComputedTrieData, HashedPostState, HashedStorage}; use std::{ @@ -646,7 +648,7 @@ mod tests { #[test] fn errors_for_unknown_parent() { - let manager = StateTrieOverlayManager::::default(); + let manager = OverlayManager::::default(); let parent = B256::random(); let anchor = B256::random(); @@ -658,7 +660,7 @@ mod tests { #[test] fn builds_managed_overlay_for_inserted_blocks() { - let manager = StateTrieOverlayManager::default(); + let manager = OverlayManager::default(); let blocks = test_blocks(); for block in &blocks { manager.insert_block(block.clone()); @@ -681,7 +683,7 @@ mod tests { #[test] fn returns_anchor_for_in_memory_parent() { - let manager = StateTrieOverlayManager::default(); + let manager = OverlayManager::default(); let blocks = test_blocks(); for block in &blocks { manager.insert_block(block.clone()); @@ -704,7 +706,7 @@ mod tests { #[test] fn prefers_anchor_in_parent_chain() { - let manager = StateTrieOverlayManager::default(); + let manager = OverlayManager::default(); let blocks = test_blocks(); for block in &blocks { manager.insert_block(block.clone()); @@ -719,7 +721,7 @@ mod tests { #[test] fn contains_hash_detects_hashes_from_anchor_to_parent() { - let manager = StateTrieOverlayManager::default(); + let manager = OverlayManager::default(); let blocks = test_blocks(); for block in &blocks { manager.insert_block(block.clone()); @@ -741,7 +743,7 @@ mod tests { #[test] fn contains_hash_rejects_hash_before_anchor() { - let manager = StateTrieOverlayManager::default(); + let manager = OverlayManager::default(); let blocks = test_blocks(); for block in &blocks { manager.insert_block(block.clone()); @@ -758,7 +760,7 @@ mod tests { #[test] fn contains_hash_rejects_unknown_anchor() { - let manager = StateTrieOverlayManager::default(); + let manager = OverlayManager::default(); let blocks = test_blocks(); for block in &blocks { manager.insert_block(block.clone()); @@ -772,7 +774,7 @@ mod tests { #[test] fn taking_sparse_trie_removes_it() { - let manager = StateTrieOverlayManager::::default(); + let manager = OverlayManager::::default(); let state_root = B256::with_last_byte(1); let other_state_root = B256::with_last_byte(2); let anchor_hash = B256::with_last_byte(3); @@ -792,7 +794,7 @@ mod tests { #[test] fn required_lookup_waits_for_in_progress_overlay() { - let manager = StateTrieOverlayManager::::default(); + let manager = OverlayManager::::default(); let key = OverlayCacheKey { anchor_hash: B256::with_last_byte(1), tip_hash: B256::with_last_byte(2), @@ -820,7 +822,7 @@ mod tests { #[test] fn prunes_cached_overlays_after_removing_blocks() { - let manager = StateTrieOverlayManager::default(); + let manager = OverlayManager::default(); let blocks = test_blocks(); for block in &blocks { manager.insert_block(block.clone()); diff --git a/crates/trie/db/Cargo.toml b/crates/trie/db/Cargo.toml index 9c6342f51a1..0600e389bab 100644 --- a/crates/trie/db/Cargo.toml +++ b/crates/trie/db/Cargo.toml @@ -20,17 +20,10 @@ reth-trie.workspace = true reth-trie-common.workspace = true reth-storage-api = { workspace = true, features = ["db-api"] } reth-storage-errors.workspace = true -reth-metrics = { workspace = true, optional = true } # alloy alloy-primitives.workspace = true -# misc -parking_lot.workspace = true - -# metrics -metrics = { workspace = true, optional = true } - # tracing tracing.workspace = true @@ -58,12 +51,11 @@ serde_json.workspace = true similar-asserts.workspace = true [features] -metrics = ["reth-trie/metrics", "dep:reth-metrics", "dep:metrics"] +metrics = ["reth-trie/metrics"] serde = [ "similar-asserts/serde", "alloy-consensus/serde", "alloy-primitives/serde", - "parking_lot/serde", "reth-storage-api/serde", "reth-trie/serde", "reth-trie-common/serde", diff --git a/crates/trie/db/src/changesets.rs b/crates/trie/db/src/changesets.rs index b1789709caa..80a2805846f 100644 --- a/crates/trie/db/src/changesets.rs +++ b/crates/trie/db/src/changesets.rs @@ -1,68 +1,30 @@ -//! Trie changeset computation and caching utilities. +//! Database-backed trie changeset computation utilities. //! -//! This module provides functionality to compute trie changesets for a given block, -//! which represent the old trie node values before the block was processed. -//! -//! It also provides an efficient in-memory cache for these changesets, which is essential for: -//! - **Reorg support**: Quickly access changesets to revert blocks during chain reorganizations -//! - **Memory efficiency**: Automatic eviction ensures bounded memory usage +//! This module reconstructs trie changesets from database state. The resulting changesets contain +//! the old trie node values needed to revert a block or contiguous range of blocks. use crate::{ DatabaseHashedCursorFactory, DatabaseStateRoot, DatabaseTrieCursorFactory, TrieTableAdapter, }; -use alloy_primitives::{map::B256Map, BlockNumber, B256}; -use parking_lot::RwLock; -use reth_primitives_traits::FastInstant as Instant; +use alloy_primitives::BlockNumber; use reth_storage_api::{ BlockNumReader, ChangeSetReader, DBProvider, StorageChangeSetReader, StorageSettingsCache, }; -use reth_storage_errors::provider::{ProviderError, ProviderResult}; -use reth_trie::{ - trie_cursor::{InMemoryTrieCursorFactory, TrieCursor, TrieCursorFactory}, - TrieInputSorted, -}; -use reth_trie_common::updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}; -use std::{ - collections::{BTreeMap, HashMap}, - ops::RangeInclusive, - sync::Arc, -}; -use tracing::{debug, warn}; - -#[cfg(test)] -use reth_trie::changesets::compute_trie_changesets; - -#[cfg(feature = "metrics")] -use reth_metrics::{ - metrics::{Counter, Gauge}, - Metrics, -}; +use reth_storage_errors::provider::ProviderError; +use reth_trie::TrieInputSorted; +use reth_trie_common::updates::TrieUpdatesSorted; +use std::{ops::RangeInclusive, sync::Arc}; +use tracing::debug; /// Computes trie changesets for a block. /// -/// # Algorithm -/// -/// For block N: -/// 1. Query cumulative `HashedPostState` revert from the database tip to after N. -/// 2. Calculate cumulative `TrieUpdates` revert from the database tip to after N. -/// 3. Query per-block `HashedPostState` revert for block N. -/// 4. Overlay the post-N trie and calculate trie updates to the pre-N state. -/// -/// # Arguments -/// -/// * `provider` - Database provider with changeset access -/// * `block_number` - Block number to compute changesets for -/// -/// # Returns -/// -/// Changesets (old trie node values) for the specified block +/// For block `N`, this reconstructs the trie as it existed after `N`, then calculates the trie +/// updates needed to restore the state before `N`. /// /// # Errors /// -/// Returns error if: -/// - Block number exceeds database tip -/// - Database access fails -/// - State root computation fails +/// Returns an error if the block exceeds the database tip, database access fails, or state root +/// computation fails. pub fn compute_block_trie_changesets( provider: &Provider, block_number: BlockNumber, @@ -84,7 +46,16 @@ where }) } -fn compute_range_trie_changesets( +/// Computes aggregate trie changesets for an inclusive block range. +/// +/// The returned changesets restore the trie from the state after `range.end()` to the state before +/// `range.start()`. `db_tip_block` must be the current database tip for `provider`. +/// +/// # Errors +/// +/// Returns an error if the range exceeds `db_tip_block`, database access fails, or state root +/// computation fails. +pub fn compute_range_trie_changesets( provider: &Provider, range: RangeInclusive, db_tip_block: BlockNumber, @@ -129,14 +100,14 @@ where } debug!( - target: "trie::changeset_cache", + target: "trie::changesets", start_block, end_block, db_tip_block, "Computing range trie changesets from database state" ); - // Step 1: collect the state revert for the requested range. + // Collect the state revert for the requested range. let range_state_revert = crate::state::from_reverts_auto(provider, range)?; let range_prefix_sets = range_state_revert.construct_prefix_sets(); @@ -147,7 +118,7 @@ where let (range_nodes, range_state) = if end_block == db_tip_block { debug!( - target: "trie::changeset_cache", + target: "trie::changesets", start_block, end_block, db_tip_block, @@ -156,14 +127,14 @@ where (Arc::default(), Arc::new(range_state_revert)) } else { - // Step 2: collect the state revert from the database tip to just after the range. + // Collect the state revert from the database tip to just after the range. let tail_state_revert = end_block .checked_add(1) .map(|next_block| crate::state::from_reverts_auto(provider, next_block..)) .transpose()? .unwrap_or_default(); - // Step 3: compute trie reverts from the database tip to just after the range. + // Compute trie reverts from the database tip to just after the range. let tail_input = TrieInputSorted::new( Arc::default(), Arc::new(tail_state_revert.clone()), @@ -177,7 +148,7 @@ where .1 .into_sorted(); - // Step 4: overlay the post-range trie and compute the trie revert to the pre-range state. + // Overlay the post-range trie and compute the trie revert to the pre-range state. let mut pre_range_state_revert = tail_state_revert; pre_range_state_revert.extend_ref_and_sort(&range_state_revert); @@ -192,7 +163,7 @@ where .into_sorted(); debug!( - target: "trie::changeset_cache", + target: "trie::changesets", start_block, end_block, num_account_nodes = range_trie_revert.account_nodes_ref().len(), @@ -202,1143 +173,3 @@ where Ok(range_trie_revert) } - -/// Computes block trie updates using the changeset cache. -/// -/// # Algorithm -/// -/// For block N: -/// 1. Get cumulative trie reverts from block N+1 to db tip using the cache -/// 2. Create an overlay cursor factory with these reverts (representing trie state after block N) -/// 3. Walk through account trie changesets for block N -/// 4. For each changed path, look up the current value using the overlay cursor -/// 5. Walk through storage trie changesets for block N -/// 6. For each changed path, look up the current value using the overlay cursor -/// 7. Return the collected trie updates -/// -/// # Arguments -/// -/// * `cache` - Handle to the changeset cache for retrieving trie reverts -/// * `provider` - Database provider for accessing changesets and block data -/// * `block_number` - Block number to compute trie updates for -/// -/// # Returns -/// -/// Trie updates representing the state of trie nodes after the block was processed -/// -/// # Errors -/// -/// Returns error if: -/// - Block number exceeds database tip -/// - Database access fails -/// - Cache retrieval fails -pub fn compute_block_trie_updates( - cache: &ChangesetCache, - provider: &Provider, - block_number: BlockNumber, -) -> ProviderResult -where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, -{ - crate::with_adapter!(provider, |A| { - compute_block_trie_updates_inner::<_, A>(cache, provider, block_number) - }) -} - -fn compute_block_trie_updates_inner( - cache: &ChangesetCache, - provider: &Provider, - block_number: BlockNumber, -) -> ProviderResult -where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - A: TrieTableAdapter, -{ - let tx = provider.tx_ref(); - - let db_tip_block = provider.best_block_number()?; - - // Step 1: Get the trie changesets for the target block from cache - let changesets = cache.get_or_compute(provider, block_number)?; - - // Step 2: Get the trie reverts for the state after the target block using the cache - let reverts = cache.get_or_compute_range(provider, (block_number + 1)..=db_tip_block)?; - - // Step 3: Create an InMemoryTrieCursorFactory with the reverts - // This gives us the trie state as it was after the target block was processed - let db_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx); - let cursor_factory = InMemoryTrieCursorFactory::new(db_cursor_factory, &reverts); - - // Step 4: Collect all account trie nodes that changed in the target block - let account_nodes_ref = changesets.account_nodes_ref(); - let mut account_nodes = Vec::with_capacity(account_nodes_ref.len()); - let mut account_cursor = cursor_factory.account_trie_cursor()?; - - // Iterate over the account nodes from the changesets - for (nibbles, _old_node) in account_nodes_ref { - // Look up the current value of this trie node using the overlay cursor - let node_value = account_cursor.seek_exact(*nibbles)?.map(|(_, node)| node); - account_nodes.push((*nibbles, node_value)); - } - - // Step 5: Collect all storage trie nodes that changed in the target block - let mut storage_tries = B256Map::default(); - - // Iterate over the storage tries from the changesets - for (hashed_address, storage_changeset) in changesets.storage_tries_ref() { - let mut storage_cursor = cursor_factory.storage_trie_cursor(*hashed_address)?; - let storage_nodes_ref = storage_changeset.storage_nodes_ref(); - let mut storage_nodes = Vec::with_capacity(storage_nodes_ref.len()); - - // Iterate over the storage nodes for this account - for (nibbles, _old_node) in storage_nodes_ref { - // Look up the current value of this storage trie node - let node_value = storage_cursor.seek_exact(*nibbles)?.map(|(_, node)| node); - storage_nodes.push((*nibbles, node_value)); - } - - storage_tries.insert( - *hashed_address, - StorageTrieUpdatesSorted { storage_nodes, is_deleted: storage_changeset.is_deleted }, - ); - } - - Ok(TrieUpdatesSorted::new(account_nodes, storage_tries)) -} - -/// Thread-safe changeset cache. -/// -/// This type wraps a shared, mutable reference to the cache inner. -/// The `RwLock` enables concurrent reads while ensuring exclusive access for writes. -#[derive(Debug, Clone)] -pub struct ChangesetCache { - inner: Arc>, -} - -impl Default for ChangesetCache { - fn default() -> Self { - Self::new() - } -} - -impl ChangesetCache { - /// Creates a new cache. - /// - /// The cache has no capacity limit and relies on explicit eviction - /// via the `evict()` method to manage memory usage. - pub fn new() -> Self { - Self { inner: Arc::new(RwLock::new(ChangesetCacheInner::new())) } - } - - /// Retrieves changesets for a block. - /// - /// Returns `None` if the block is not in the cache (either evicted or never computed). - /// Updates hit/miss metrics accordingly. - pub fn get( - &self, - block_hash: B256, - block_number: BlockNumber, - ) -> Option> { - self.inner.read().get(&ChangesetRangeKey::single(block_number, block_hash)) - } - - /// Evicts changesets for blocks below the given block number. - /// - /// This should be called after blocks are persisted to the database to free - /// memory for changesets that are no longer needed in the cache. - /// - /// # Arguments - /// - /// * `up_to_block` - Evict blocks with number < this value. Blocks with number >= this value - /// are retained. - pub fn evict(&self, up_to_block: BlockNumber) { - self.inner.write().evict(up_to_block) - } - - /// Gets changesets from cache, or computes them on-the-fly if missing. - /// - /// This is the primary API for retrieving changesets. It checks the cache first, then falls - /// back to computing from database state if missing. - /// - /// # Arguments - /// - /// * `block_number` - Block number (for cache insertion and logging) - /// * `provider` - Database provider for DB access - /// - /// # Returns - /// - /// Changesets for the block, either from cache or computed on-the-fly. - pub fn get_or_compute

( - &self, - provider: &P, - block_number: BlockNumber, - ) -> ProviderResult> - where - P: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - { - self.get_or_compute_range(provider, block_number..=block_number) - } - - /// Gets or computes trie reverts for a range of blocks. - /// - /// If all blocks in the range are cached, this method retrieves and accumulates those - /// per-block trie changesets (reverts) in reverse order (newest to oldest), so that older - /// values take precedence when there are conflicts. - /// - /// If any block is missing from cache, this falls back to one aggregate database computation - /// for the whole range. The aggregate result restores the trie to the state before the range - /// and is inserted into the range cache. - /// - /// # Arguments - /// - /// * `provider` - Database provider for DB access and block lookups - /// * `range` - Block range to accumulate reverts for (inclusive) - /// - /// # Returns - /// - /// Accumulated trie reverts for all blocks in the specified range - /// - /// # Errors - /// - /// Returns error if: - /// - Any block in the range is beyond the database tip - /// - Database access fails - /// - Block hash lookup fails - /// - Changeset computation fails - pub fn get_or_compute_range

( - &self, - provider: &P, - range: RangeInclusive, - ) -> ProviderResult> - where - P: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - { - let db_tip_block = provider.best_block_number()?; - - let start_block = *range.start(); - let end_block = *range.end(); - - // If range end is beyond the tip, return an error - if end_block > db_tip_block { - return Err(ProviderError::InsufficientChangesets { - requested: end_block, - available: 0..=db_tip_block, - }); - } - - let timer = Instant::now(); - - debug!( - target: "trie::changeset_cache", - start_block, - end_block, - db_tip_block, - "Starting get_or_compute_range" - ); - - if start_block > end_block { - debug!( - target: "trie::changeset_cache", - start_block, - end_block, - "Empty changeset range requested" - ); - return Ok(Arc::new(TrieUpdatesSorted::default())) - } - - let end_block_hash = provider.block_hash(end_block)?.ok_or_else(|| { - ProviderError::other(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("block hash not found for block number {}", end_block), - )) - })?; - let range_key = ChangesetRangeKey::new(start_block, end_block, end_block_hash); - - if let Some(accumulated_reverts) = self.inner.read().get(&range_key) { - let elapsed = timer.elapsed(); - - debug!( - target: "trie::changeset_cache", - ?elapsed, - start_block, - end_block, - ?end_block_hash, - num_blocks = end_block.saturating_sub(start_block).saturating_add(1), - "Changeset cache HIT for block range" - ); - - return Ok(accumulated_reverts) - } - - let mut cached_reverts = - Vec::with_capacity(end_block.saturating_sub(start_block).saturating_add(1) as usize); - let mut all_cached = true; - - for block_number in range.rev() { - // Get the block hash for this block number - let block_hash = if block_number == end_block { - end_block_hash - } else { - provider.block_hash(block_number)?.ok_or_else(|| { - ProviderError::other(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("block hash not found for block number {}", block_number), - )) - })? - }; - - debug!( - target: "trie::changeset_cache", - block_number, - ?block_hash, - "Looked up block hash for block number in range" - ); - - let block_key = ChangesetRangeKey::single(block_number, block_hash); - if let Some(changesets) = self.inner.read().get(&block_key) { - cached_reverts.push(changesets); - } else { - all_cached = false; - break - } - } - - if all_cached { - // `merge_slice` gives precedence to earlier items, so pass reverts oldest-to-newest. - cached_reverts.reverse(); - let accumulated_reverts = Arc::new(TrieUpdatesSorted::merge_slice(&cached_reverts)); - let elapsed = timer.elapsed(); - - let num_account_nodes = accumulated_reverts.account_nodes_ref().len(); - let num_storage_tries = accumulated_reverts.storage_tries_ref().len(); - - debug!( - target: "trie::changeset_cache", - ?elapsed, - start_block, - end_block, - num_blocks = end_block.saturating_sub(start_block).saturating_add(1), - num_account_nodes, - num_storage_tries, - "Finished accumulating cached trie reverts for block range" - ); - - self.inner.write().insert(range_key, Arc::clone(&accumulated_reverts)); - return Ok(accumulated_reverts) - } - - warn!( - target: "trie::changeset_cache", - start_block, - end_block, - "Changeset cache MISS in range, falling back to aggregate DB-based computation" - ); - - let accumulated_reverts = Arc::new(compute_range_trie_changesets( - provider, - start_block..=end_block, - db_tip_block, - )?); - - let elapsed = timer.elapsed(); - - let num_account_nodes = accumulated_reverts.account_nodes_ref().len(); - let num_storage_tries = accumulated_reverts.storage_tries_ref().len(); - - debug!( - target: "trie::changeset_cache", - ?elapsed, - start_block, - end_block, - ?end_block_hash, - num_blocks = end_block.saturating_sub(start_block).saturating_add(1), - num_account_nodes, - num_storage_tries, - "Finished accumulating trie reverts for block range" - ); - - self.inner.write().insert(range_key, Arc::clone(&accumulated_reverts)); - - Ok(accumulated_reverts) - } -} - -/// Cache key for one contiguous range of canonical trie changesets. -/// -/// The end block hash disambiguates canonical rewrites where the same block numbers later refer to -/// a different chain. For a single block, `start_block == end_block` and `end_block_hash` is that -/// block's hash. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] -struct ChangesetRangeKey { - start_block: BlockNumber, - end_block: BlockNumber, - end_block_hash: B256, -} - -impl ChangesetRangeKey { - const fn new(start_block: BlockNumber, end_block: BlockNumber, end_block_hash: B256) -> Self { - Self { start_block, end_block, end_block_hash } - } - - const fn single(block_number: BlockNumber, block_hash: B256) -> Self { - Self::new(block_number, block_number, block_hash) - } -} - -/// In-memory cache for trie changesets with explicit eviction policy. -/// -/// Holds changesets for blocks or block ranges that have been validated but not yet persisted. -/// Keyed by canonical block range. Eviction is controlled -/// explicitly by the engine API tree handler when persistence completes. -/// -/// ## Eviction Policy -/// -/// Unlike traditional caches with automatic eviction, this cache requires explicit -/// eviction calls. The engine API tree handler calls `evict(block_number)` after -/// blocks are persisted to the database, ensuring changesets remain available -/// until their corresponding blocks are safely on disk. -/// -/// ## Metrics -/// -/// The cache maintains several metrics for observability: -/// - `hits`: Number of successful cache lookups -/// - `misses`: Number of failed cache lookups -/// - `evictions`: Number of blocks evicted -/// - `size`: Current number of cached blocks -#[derive(Debug)] -struct ChangesetCacheInner { - /// Cache entries keyed by inclusive block range plus the range's canonical end hash. - entries: HashMap>, - - /// Range start block to cache keys mapping for eviction. - range_starts: BTreeMap>, - - /// Metrics for monitoring cache behavior - #[cfg(feature = "metrics")] - metrics: ChangesetCacheMetrics, -} - -#[cfg(feature = "metrics")] -/// Metrics for the changeset cache. -/// -/// These metrics provide visibility into cache performance and help identify -/// potential issues like high miss rates. -#[derive(Metrics, Clone)] -#[metrics(scope = "trie.changeset_cache")] -struct ChangesetCacheMetrics { - /// Cache hit counter - hits: Counter, - - /// Cache miss counter - misses: Counter, - - /// Eviction counter - evictions: Counter, - - /// Current cache size (number of entries) - size: Gauge, -} - -impl Default for ChangesetCacheInner { - fn default() -> Self { - Self::new() - } -} - -impl ChangesetCacheInner { - /// Creates a new empty changeset cache. - /// - /// The cache has no capacity limit and relies on explicit eviction - /// via the `evict()` method to manage memory usage. - fn new() -> Self { - Self { - entries: HashMap::new(), - range_starts: BTreeMap::new(), - #[cfg(feature = "metrics")] - metrics: Default::default(), - } - } - - fn get(&self, key: &ChangesetRangeKey) -> Option> { - match self.entries.get(key) { - Some(changesets) => { - #[cfg(feature = "metrics")] - self.metrics.hits.increment(1); - Some(Arc::clone(changesets)) - } - None => { - #[cfg(feature = "metrics")] - self.metrics.misses.increment(1); - None - } - } - } - - fn insert(&mut self, key: ChangesetRangeKey, changesets: Arc) { - debug!( - target: "trie::changeset_cache", - ?key, - cache_size_before = self.entries.len(), - "Inserting changeset into cache" - ); - - let is_new_entry = self.entries.insert(key, changesets).is_none(); - - if is_new_entry { - self.range_starts.entry(key.start_block).or_default().push(key); - } - - // Update size metric - #[cfg(feature = "metrics")] - self.metrics.size.set(self.entries.len() as f64); - - debug!( - target: "trie::changeset_cache", - ?key, - cache_size_after = self.entries.len(), - "Changeset inserted into cache" - ); - } - - fn evict(&mut self, up_to_block: BlockNumber) { - debug!( - target: "trie::changeset_cache", - up_to_block, - cache_size_before = self.entries.len(), - "Starting cache eviction" - ); - - // Find all block numbers that should be evicted (< up_to_block) - let range_starts_to_evict: Vec = - self.range_starts.range(..up_to_block).map(|(num, _)| *num).collect(); - - // Remove entries for each block number below threshold - #[cfg(feature = "metrics")] - let mut evicted_count = 0; - #[cfg(not(feature = "metrics"))] - let mut evicted_count = 0; - - for start_block in &range_starts_to_evict { - if let Some(keys) = self.range_starts.remove(start_block) { - debug!( - target: "trie::changeset_cache", - start_block, - num_ranges = keys.len(), - "Evicting ranges from cache" - ); - for key in keys { - if self.entries.remove(&key).is_some() { - evicted_count += 1; - } - } - } - } - - debug!( - target: "trie::changeset_cache", - up_to_block, - evicted_count, - cache_size_after = self.entries.len(), - "Finished cache eviction" - ); - - // Update metrics if we evicted anything - #[cfg(feature = "metrics")] - if evicted_count > 0 { - self.metrics.evictions.increment(evicted_count as u64); - self.metrics.size.set(self.entries.len() as f64); - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use alloy_consensus::Header; - use alloy_primitives::{ - keccak256, - map::{B256Map, HashMap}, - Address, U256, - }; - use reth_db::tables; - use reth_db_api::{ - models::{AccountBeforeTx, BlockNumberAddress}, - transaction::DbTxMut, - }; - use reth_primitives_traits::{Account, StorageEntry}; - use reth_provider::{ - test_utils::create_test_provider_factory, StaticFileProviderFactory, StaticFileSegment, - StaticFileWriter, - }; - use reth_stages_types::{StageCheckpoint, StageId}; - use reth_storage_api::{StageCheckpointWriter, TrieWriter}; - use reth_trie::{BranchNodeCompact, Nibbles, StateRoot}; - - // Helper function to create empty TrieUpdatesSorted for testing - fn create_test_changesets() -> Arc { - Arc::new(TrieUpdatesSorted::new(vec![], B256Map::default())) - } - - fn insert_test_changesets( - cache: &mut ChangesetCacheInner, - block_hash: B256, - block_number: BlockNumber, - changesets: Arc, - ) { - cache.insert(ChangesetRangeKey::single(block_number, block_hash), changesets); - } - - fn get_test_changesets( - cache: &ChangesetCacheInner, - block_hash: B256, - block_number: BlockNumber, - ) -> Option> { - cache.get(&ChangesetRangeKey::single(block_number, block_hash)) - } - - fn test_account(balance: u64) -> Account { - Account { balance: U256::from(balance), ..Default::default() } - } - - fn test_storage(slot: u64, value: u64) -> StorageEntry { - StorageEntry { key: B256::from(U256::from(slot)), value: U256::from(value) } - } - - fn seed_headers( - factory: &impl StaticFileProviderFactory< - Primitives: reth_primitives_traits::NodePrimitives, - >, - end_block: BlockNumber, - ) { - let static_file_provider = factory.static_file_provider(); - let mut header_writer = - static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); - for block_number in 0..=end_block { - let header = Header { number: block_number, ..Default::default() }; - header_writer - .append_header(&header, &B256::with_last_byte(block_number as u8)) - .unwrap(); - } - header_writer.commit().unwrap(); - } - - fn legacy_compute_range_trie_changesets( - provider: &Provider, - range: RangeInclusive, - ) -> TrieUpdatesSorted - where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - { - let mut accumulated_reverts = TrieUpdatesSorted::default(); - for block_number in range.rev() { - let changesets = legacy_compute_block_trie_changesets(provider, block_number); - accumulated_reverts.extend_ref_and_sort(&changesets); - } - accumulated_reverts - } - - fn legacy_compute_block_trie_changesets( - provider: &Provider, - block_number: BlockNumber, - ) -> TrieUpdatesSorted - where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - { - crate::with_adapter!(provider, |A| { - legacy_compute_block_trie_changesets_inner::<_, A>(provider, block_number) - }) - } - - fn legacy_compute_block_trie_changesets_inner( - provider: &Provider, - block_number: BlockNumber, - ) -> TrieUpdatesSorted - where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - A: TrieTableAdapter, - { - let individual_state_revert = - crate::state::from_reverts_auto(provider, block_number..=block_number).unwrap(); - let cumulative_state_revert = - crate::state::from_reverts_auto(provider, (block_number + 1)..).unwrap(); - - let mut cumulative_state_revert_prev = cumulative_state_revert.clone(); - cumulative_state_revert_prev.extend_ref_and_sort(&individual_state_revert); - - type DbStateRoot<'a, TX, A> = - StateRoot, DatabaseHashedCursorFactory<&'a TX>>; - - let input_prev = TrieInputSorted::new( - Arc::default(), - Arc::new(cumulative_state_revert_prev.clone()), - cumulative_state_revert_prev.construct_prefix_sets(), - ); - let cumulative_trie_updates_prev = - DbStateRoot::<_, A>::overlay_root_from_nodes_with_updates( - provider.tx_ref(), - input_prev, - ) - .unwrap() - .1 - .into_sorted(); - - let input = TrieInputSorted::new( - Arc::new(cumulative_trie_updates_prev.clone()), - Arc::new(cumulative_state_revert), - individual_state_revert.construct_prefix_sets(), - ); - let trie_updates = - DbStateRoot::<_, A>::overlay_root_from_nodes_with_updates(provider.tx_ref(), input) - .unwrap() - .1 - .into_sorted(); - - let db_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(provider.tx_ref()); - let overlay_factory = - InMemoryTrieCursorFactory::new(db_cursor_factory, &cumulative_trie_updates_prev); - - compute_trie_changesets(&overlay_factory, &trie_updates).unwrap() - } - - fn seed_tip_trie_tables(provider: &Provider) - where - Provider: DBProvider + TrieWriter, - A: TrieTableAdapter, - { - type DbStateRoot<'a, TX, A> = - StateRoot, DatabaseHashedCursorFactory<&'a TX>>; - - let (_, trie_updates) = - DbStateRoot::<_, A>::from_tx(provider.tx_ref()).root_with_updates().unwrap(); - provider.write_trie_updates(trie_updates).unwrap(); - } - - #[test] - fn cached_range_merge_keeps_oldest_revert_values() { - let factory = create_test_provider_factory(); - seed_headers(&factory, 2); - - let provider = factory.provider_rw().unwrap(); - provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(2)).unwrap(); - - let cache = ChangesetCache::new(); - let path = Nibbles::from_nibbles([0x1, 0x2]); - let older_node = BranchNodeCompact::new(0b0001, 0, 0, vec![], None); - let newer_node = BranchNodeCompact::new(0b0010, 0, 0, vec![], None); - - { - let mut cache = cache.inner.write(); - insert_test_changesets( - &mut cache, - B256::with_last_byte(1), - 1, - Arc::new(TrieUpdatesSorted::new( - vec![(path, Some(older_node.clone()))], - B256Map::default(), - )), - ); - insert_test_changesets( - &mut cache, - B256::with_last_byte(2), - 2, - Arc::new(TrieUpdatesSorted::new( - vec![(path, Some(newer_node))], - B256Map::default(), - )), - ); - } - - let accumulated = cache.get_or_compute_range(&*provider, 1..=2).unwrap(); - assert_eq!(accumulated.account_nodes_ref(), &[(path, Some(older_node))]); - } - - #[test] - fn aggregate_range_reverts_to_pre_range_state() { - let factory = create_test_provider_factory(); - seed_headers(&factory, 3); - - let provider = factory.provider_rw().unwrap(); - let address = Address::with_last_byte(1); - let hashed_address = keccak256(address); - let slot1 = B256::from(U256::from(1)); - let slot2 = B256::from(U256::from(2)); - let account1 = test_account(10); - let account2 = test_account(20); - let account3 = test_account(30); - - provider.tx_ref().put::(hashed_address, account3).unwrap(); - provider - .tx_ref() - .put::( - hashed_address, - StorageEntry { key: keccak256(slot1), value: U256::from(25) }, - ) - .unwrap(); - provider - .tx_ref() - .put::( - hashed_address, - StorageEntry { key: keccak256(slot2), value: U256::from(20) }, - ) - .unwrap(); - - provider - .tx_ref() - .put::(1, AccountBeforeTx { address, info: None }) - .unwrap(); - provider - .tx_ref() - .put::(2, AccountBeforeTx { address, info: Some(account1) }) - .unwrap(); - provider - .tx_ref() - .put::(3, AccountBeforeTx { address, info: Some(account2) }) - .unwrap(); - - provider - .tx_ref() - .put::(BlockNumberAddress((1, address)), test_storage(1, 0)) - .unwrap(); - provider - .tx_ref() - .put::(BlockNumberAddress((1, address)), test_storage(2, 0)) - .unwrap(); - provider - .tx_ref() - .put::( - BlockNumberAddress((2, address)), - StorageEntry { key: slot1, value: U256::from(10) }, - ) - .unwrap(); - provider - .tx_ref() - .put::( - BlockNumberAddress((3, address)), - StorageEntry { key: slot1, value: U256::from(15) }, - ) - .unwrap(); - - provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(3)).unwrap(); - crate::with_adapter!(provider, |A| seed_tip_trie_tables::<_, A>(&*provider)); - - let actual = compute_range_trie_changesets(&*provider, 1..=3, 3).unwrap(); - let storage_revert = actual - .storage_tries_ref() - .get(&hashed_address) - .expect("created account storage trie should be deleted by range revert"); - assert!(storage_revert.is_deleted()); - assert!(storage_revert.storage_nodes_ref().is_empty()); - - let cache = ChangesetCache::new(); - let from_cache_api = cache.get_or_compute_range(&*provider, 1..=3).unwrap(); - assert_eq!(*from_cache_api, actual); - assert_eq!(cache.inner.read().entries.len(), 1); - - let block_changesets = cache.get_or_compute(&*provider, 2).unwrap(); - assert_eq!(*block_changesets, legacy_compute_block_trie_changesets(&*provider, 2)); - assert_eq!(cache.inner.read().entries.len(), 2); - } - - #[test] - fn aggregate_range_matches_legacy_per_block_merge_with_storage_wipe() { - let factory = create_test_provider_factory(); - seed_headers(&factory, 3); - - let provider = factory.provider_rw().unwrap(); - let address = Address::with_last_byte(1); - let slot1 = B256::from(U256::from(1)); - let slot2 = B256::from(U256::from(2)); - let account1 = test_account(10); - let account2 = test_account(20); - - provider - .tx_ref() - .put::(1, AccountBeforeTx { address, info: None }) - .unwrap(); - provider - .tx_ref() - .put::(2, AccountBeforeTx { address, info: Some(account1) }) - .unwrap(); - provider - .tx_ref() - .put::(3, AccountBeforeTx { address, info: Some(account2) }) - .unwrap(); - - provider - .tx_ref() - .put::(BlockNumberAddress((1, address)), test_storage(1, 0)) - .unwrap(); - provider - .tx_ref() - .put::(BlockNumberAddress((1, address)), test_storage(2, 0)) - .unwrap(); - provider - .tx_ref() - .put::( - BlockNumberAddress((2, address)), - StorageEntry { key: slot1, value: U256::from(10) }, - ) - .unwrap(); - provider - .tx_ref() - .put::( - BlockNumberAddress((3, address)), - StorageEntry { key: slot1, value: U256::from(15) }, - ) - .unwrap(); - provider - .tx_ref() - .put::( - BlockNumberAddress((3, address)), - StorageEntry { key: slot2, value: U256::from(20) }, - ) - .unwrap(); - - provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(3)).unwrap(); - crate::with_adapter!(provider, |A| seed_tip_trie_tables::<_, A>(&*provider)); - - let expected = legacy_compute_range_trie_changesets(&*provider, 2..=3); - let actual = compute_range_trie_changesets(&*provider, 2..=3, 3).unwrap(); - assert_eq!(actual, expected); - } - - #[test] - fn test_insert_and_retrieve_single_entry() { - let mut cache = ChangesetCacheInner::new(); - let hash = B256::random(); - let changesets = create_test_changesets(); - - insert_test_changesets(&mut cache, hash, 100, Arc::clone(&changesets)); - - // Should be able to retrieve it - let retrieved = get_test_changesets(&cache, hash, 100); - assert!(retrieved.is_some()); - assert_eq!(cache.entries.len(), 1); - } - - #[test] - fn test_insert_multiple_entries() { - let mut cache = ChangesetCacheInner::new(); - - // Insert 10 blocks - let mut hashes = Vec::new(); - for i in 0..10 { - let hash = B256::random(); - insert_test_changesets(&mut cache, hash, 100 + i, create_test_changesets()); - hashes.push((100 + i, hash)); - } - - // Should be able to retrieve all - assert_eq!(cache.entries.len(), 10); - for (block_number, hash) in hashes { - assert!(get_test_changesets(&cache, hash, block_number).is_some()); - } - } - - #[test] - fn test_eviction_when_explicitly_called() { - let mut cache = ChangesetCacheInner::new(); - - // Insert 15 blocks (0-14) - let mut hashes = Vec::new(); - for i in 0..15 { - let hash = B256::random(); - insert_test_changesets(&mut cache, hash, i, create_test_changesets()); - hashes.push((i, hash)); - } - - // All blocks should be present (no automatic eviction) - assert_eq!(cache.entries.len(), 15); - - // Explicitly evict blocks < 4 - cache.evict(4); - - // Blocks 0-3 should be evicted - assert_eq!(cache.entries.len(), 11); // blocks 4-14 = 11 blocks - - // Verify blocks 0-3 are evicted - for i in 0..4 { - assert!( - get_test_changesets(&cache, hashes[i as usize].1, i).is_none(), - "Block {} should be evicted", - i - ); - } - - // Verify blocks 4-14 are still present - for i in 4..15 { - assert!( - get_test_changesets(&cache, hashes[i as usize].1, i).is_some(), - "Block {} should be present", - i - ); - } - } - - #[test] - fn test_eviction_with_persistence_watermark() { - let mut cache = ChangesetCacheInner::new(); - - // Insert blocks 100-165 - let mut hashes = HashMap::new(); - for i in 100..=165 { - let hash = B256::random(); - insert_test_changesets(&mut cache, hash, i, create_test_changesets()); - hashes.insert(i, hash); - } - - // All blocks should be present (no automatic eviction) - assert_eq!(cache.entries.len(), 66); - - // Simulate persistence up to block 164, with 64-block retention window - // Eviction threshold = 164 - 64 = 100 - cache.evict(100); - - // Blocks 100-165 should remain (66 blocks) - assert_eq!(cache.entries.len(), 66); - - // Simulate persistence up to block 165 - // Eviction threshold = 165 - 64 = 101 - cache.evict(101); - - // Blocks 101-165 should remain (65 blocks) - assert_eq!(cache.entries.len(), 65); - assert!(get_test_changesets(&cache, hashes[&100], 100).is_none()); - assert!(get_test_changesets(&cache, hashes[&101], 101).is_some()); - } - - #[test] - fn test_out_of_order_inserts_with_explicit_eviction() { - let mut cache = ChangesetCacheInner::new(); - - // Insert blocks in random order - let hash_10 = B256::random(); - insert_test_changesets(&mut cache, hash_10, 10, create_test_changesets()); - - let hash_5 = B256::random(); - insert_test_changesets(&mut cache, hash_5, 5, create_test_changesets()); - - let hash_15 = B256::random(); - insert_test_changesets(&mut cache, hash_15, 15, create_test_changesets()); - - let hash_3 = B256::random(); - insert_test_changesets(&mut cache, hash_3, 3, create_test_changesets()); - - // All blocks should be present (no automatic eviction) - assert_eq!(cache.entries.len(), 4); - - // Explicitly evict blocks < 5 - cache.evict(5); - - assert!(get_test_changesets(&cache, hash_3, 3).is_none(), "Block 3 should be evicted"); - assert!(get_test_changesets(&cache, hash_5, 5).is_some(), "Block 5 should be present"); - assert!(get_test_changesets(&cache, hash_10, 10).is_some(), "Block 10 should be present"); - assert!(get_test_changesets(&cache, hash_15, 15).is_some(), "Block 15 should be present"); - } - - #[test] - fn test_multiple_blocks_same_number() { - let mut cache = ChangesetCacheInner::new(); - - // Insert multiple blocks with same number (side chains) - let hash_1a = B256::random(); - let hash_1b = B256::random(); - insert_test_changesets(&mut cache, hash_1a, 100, create_test_changesets()); - insert_test_changesets(&mut cache, hash_1b, 100, create_test_changesets()); - - // Both should be retrievable - assert!(get_test_changesets(&cache, hash_1a, 100).is_some()); - assert!(get_test_changesets(&cache, hash_1b, 100).is_some()); - assert_eq!(cache.entries.len(), 2); - } - - #[test] - fn test_ranges_with_same_numbers_and_different_end_hashes_are_distinct() { - let mut cache = ChangesetCacheInner::new(); - let path = Nibbles::from_nibbles_unchecked([0x01]); - let hash_a = B256::with_last_byte(1); - let hash_b = B256::with_last_byte(2); - let key_a = ChangesetRangeKey::new(10, 20, hash_a); - let key_b = ChangesetRangeKey::new(10, 20, hash_b); - let changesets_a = Arc::new(TrieUpdatesSorted::new( - vec![(path, Some(BranchNodeCompact::new(0b0001, 0, 0, vec![], None)))], - B256Map::default(), - )); - let changesets_b = Arc::new(TrieUpdatesSorted::new( - vec![(path, Some(BranchNodeCompact::new(0b0010, 0, 0, vec![], None)))], - B256Map::default(), - )); - - cache.insert(key_a, Arc::clone(&changesets_a)); - cache.insert(key_b, Arc::clone(&changesets_b)); - - assert_eq!(cache.entries.len(), 2); - assert_eq!( - cache.get(&key_a).unwrap().account_nodes_ref(), - changesets_a.account_nodes_ref() - ); - assert_eq!( - cache.get(&key_b).unwrap().account_nodes_ref(), - changesets_b.account_nodes_ref() - ); - - cache.evict(11); - assert!(cache.get(&key_a).is_none()); - assert!(cache.get(&key_b).is_none()); - } - - #[test] - fn test_eviction_removes_all_side_chains() { - let mut cache = ChangesetCacheInner::new(); - - // Insert multiple blocks at same height (side chains) - let hash_10a = B256::random(); - let hash_10b = B256::random(); - let hash_10c = B256::random(); - insert_test_changesets(&mut cache, hash_10a, 10, create_test_changesets()); - insert_test_changesets(&mut cache, hash_10b, 10, create_test_changesets()); - insert_test_changesets(&mut cache, hash_10c, 10, create_test_changesets()); - - let hash_20 = B256::random(); - insert_test_changesets(&mut cache, hash_20, 20, create_test_changesets()); - - assert_eq!(cache.entries.len(), 4); - - // Evict blocks < 15 - should remove all three side chains at height 10 - cache.evict(15); - - assert_eq!(cache.entries.len(), 1); - assert!(get_test_changesets(&cache, hash_10a, 10).is_none()); - assert!(get_test_changesets(&cache, hash_10b, 10).is_none()); - assert!(get_test_changesets(&cache, hash_10c, 10).is_none()); - assert!(get_test_changesets(&cache, hash_20, 20).is_some()); - } -} diff --git a/crates/trie/parallel/Cargo.toml b/crates/trie/parallel/Cargo.toml index de49c364e7e..b9bc03c7513 100644 --- a/crates/trie/parallel/Cargo.toml +++ b/crates/trie/parallel/Cargo.toml @@ -50,7 +50,7 @@ reth-chainspec.workspace = true reth-ethereum-primitives.workspace = true reth-primitives-traits.workspace = true reth-provider = { workspace = true, features = ["test-utils"] } -reth-trie-db.workspace = true +reth-storage-overlay.workspace = true reth-trie = { workspace = true, features = ["test-utils"] } # misc @@ -68,7 +68,6 @@ trie-debug = [ test-utils = [ "reth-primitives-traits/test-utils", "reth-provider/test-utils", - "reth-trie-db/test-utils", "reth-trie/test-utils", "reth-tasks/test-utils", "reth-chainspec/test-utils", diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index 4d43033946b..06da929ab00 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -1173,10 +1173,10 @@ mod tests { let chain_spec = Arc::new(ChainSpec::default()); let anchor_hash = chain_spec.genesis_hash(); let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec); - let changeset_cache = reth_trie_db::ChangesetCache::new(); + let changeset_cache = reth_storage_overlay::ChangesetCache::new(); let factory = reth_provider::providers::OverlayStateProviderFactory::new( provider_factory, - reth_provider::providers::OverlayBuilder::::new( + reth_storage_overlay::OverlayBuilder::::new( anchor_hash, changeset_cache, ), diff --git a/examples/custom-state-root/Cargo.toml b/examples/custom-state-root/Cargo.toml index 9e9c71f304d..b7601d10b38 100644 --- a/examples/custom-state-root/Cargo.toml +++ b/examples/custom-state-root/Cargo.toml @@ -9,9 +9,9 @@ license.workspace = true reth-ethereum = { workspace = true, features = ["node", "test-utils"] } reth-engine-tree.workspace = true reth-evm.workspace = true -reth-chain-state.workspace = true reth-primitives-traits.workspace = true reth-provider.workspace = true +reth-storage-overlay.workspace = true reth-trie.workspace = true eyre.workspace = true diff --git a/examples/custom-state-root/src/main.rs b/examples/custom-state-root/src/main.rs index 813969dc8ea..fc9cad260fc 100644 --- a/examples/custom-state-root/src/main.rs +++ b/examples/custom-state-root/src/main.rs @@ -22,7 +22,6 @@ use std::sync::Arc; use alloy_genesis::Genesis; use alloy_primitives::B256; -use reth_chain_state::StateTrieOverlayManager; use reth_engine_tree::tree::{ state_root_strategy::{ DefaultStateRootStrategy, LazyHashedPostState, PayloadStateRootHandle, @@ -36,8 +35,8 @@ use reth_ethereum::{ node::{ builder::{ rpc::{ - BasicEngineApiBuilder, BasicEngineValidatorBuilder, ChangesetCache, - EngineValidatorBuilder, Identity, RpcAddOns, + BasicEngineApiBuilder, BasicEngineValidatorBuilder, EngineValidatorBuilder, + Identity, RpcAddOns, }, FullNodeComponents, NodeBuilder, NodeHandle, }, @@ -50,6 +49,7 @@ use reth_ethereum::{ use reth_evm::{revm::context::Block as _, ConfigureEvm}; use reth_primitives_traits::{NodePrimitives, RecoveredBlock}; use reth_provider::{BlockExecutionOutput, ProviderResult}; +use reth_storage_overlay::{ChangesetCache, OverlayManager}; use reth_trie::updates::TrieUpdates; /// Strategy that returns `B256::ZERO` as the state root from an activation timestamp on, and @@ -149,7 +149,7 @@ where ctx: &reth_ethereum::node::builder::AddOnsContext<'_, N>, tree_config: TreeConfig, changeset_cache: ChangesetCache, - state_trie_overlays: StateTrieOverlayManager, + state_trie_overlays: OverlayManager, ) -> eyre::Result { let validator = self .inner From d3ddd586618b5a108c097d8d7a982d44dbd435b0 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Mon, 27 Jul 2026 19:30:48 +0300 Subject: [PATCH 052/148] docs(engine): document sparse trie empty account invariant (#26508) --- .../tree/src/tree/state_root_strategy/mod.rs | 8 +++++++ .../tree/state_root_strategy/sparse_trie.rs | 21 ++++++++++++++++++- 2 files changed, 28 insertions(+), 1 deletion(-) diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 13e7c8327ce..6e15736b43b 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -41,6 +41,14 @@ //! blocks where the default behavior is wanted, for example before a fork activates. See //! `examples/custom-state-root` for the wiring. //! +//! # Empty accounts +//! +//! The sparse-trie path treats an account with zero nonce, zero balance, empty bytecode, and an +//! empty storage root as absent from the account trie. This relies on the post-Merge invariant +//! established by EIP-7523 (): post-Merge state cannot +//! contain empty accounts. A custom strategy that replays pre-Merge blocks through the Engine API +//! must route those blocks to a state-root implementation that supports historical empty accounts. +//! //! Returning empty trie updates in the outcome means the trie tables are no longer maintained: //! `eth_getProof` and anything else that reads the stored trie will not work for new blocks. //! Sparse-trie cache pruning uses node epochs to retain the in-memory block range. diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 1840e9e3405..e754a07577a 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -1046,6 +1046,11 @@ fn dispatch_with_chunking( } /// RLP-encodes the account as a [`TrieAccount`] leaf value, or returns empty for deletions. +/// +/// `Some(Account::default())` with an empty storage root is encoded as a deletion. This is valid +/// for post-Merge state because EIP-7523 () prohibits +/// empty accounts. Do not use this encoding rule when replaying historical pre-Merge state, where +/// an empty account and a missing account can have different trie representations. fn encode_account_leaf_value( account: Option, storage_root: B256, @@ -1171,7 +1176,7 @@ mod tests { } #[test] - fn test_encode_account_leaf_value_empty_account_and_empty_root_is_empty() { + fn test_encode_account_leaf_value_deletion_and_empty_root_is_empty() { let mut account_rlp_buf = vec![0xAB]; let encoded = encode_account_leaf_value(None, EMPTY_ROOT_HASH, &mut account_rlp_buf); @@ -1180,6 +1185,20 @@ mod tests { assert_eq!(account_rlp_buf, vec![0xAB]); } + #[test] + fn test_encode_account_leaf_value_empty_account_and_empty_root_is_empty() { + let mut account_rlp_buf = vec![0xAB]; + let encoded = encode_account_leaf_value( + Some(Account::default()), + EMPTY_ROOT_HASH, + &mut account_rlp_buf, + ); + + assert!(encoded.is_empty()); + // Early return should not touch the caller's buffer. + assert_eq!(account_rlp_buf, vec![0xAB]); + } + #[test] fn test_encode_account_leaf_value_non_empty_account_is_rlp() { let storage_root = B256::from([0x99; 32]); From fc3f7da91b345cd011767a3e71cde541ebfa02e0 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Mon, 27 Jul 2026 19:16:56 +0200 Subject: [PATCH 053/148] perf: share sender recovery cache across transaction paths (#26494) Co-authored-by: Alexey Shekhirin --- .github/workflows/docker.yml | 1 + Cargo.lock | 2 + Cargo.toml | 2 +- Dockerfile.depot | 2 + crates/ethereum/evm/src/lib.rs | 21 ++++- crates/ethereum/node/src/node.rs | 5 +- crates/evm/evm/Cargo.toml | 2 + crates/evm/evm/src/lib.rs | 2 + crates/evm/evm/src/sender_recovery.rs | 90 ++++++++++++++++++++++ crates/net/network/Cargo.toml | 2 + crates/net/network/src/builder.rs | 9 +++ crates/net/network/src/transactions/mod.rs | 37 ++++++--- crates/node/builder/src/builder/mod.rs | 23 +++++- crates/node/core/src/args/engine.rs | 32 ++++++++ crates/transaction-pool/src/traits.rs | 11 +++ docs/vocs/docs/pages/cli/reth/node.mdx | 5 ++ 16 files changed, 229 insertions(+), 17 deletions(-) create mode 100644 crates/evm/evm/src/sender_recovery.rs diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 7d8264745e0..cdd6b24737e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -95,6 +95,7 @@ jobs: echo "ethereum_set<> "$GITHUB_OUTPUT" diff --git a/Cargo.lock b/Cargo.lock index f7d7054af59..b96c4c7ba56 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8741,6 +8741,7 @@ dependencies = [ "alloy-primitives", "auto_impl", "derive_more", + "fixed-cache", "futures-util", "metrics", "rayon", @@ -9080,6 +9081,7 @@ dependencies = [ "reth-eth-wire-types", "reth-ethereum-forks", "reth-ethereum-primitives", + "reth-evm", "reth-evm-ethereum", "reth-fs-util", "reth-metrics", diff --git a/Cargo.toml b/Cargo.toml index beb4be10fc2..9bf45de6693 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -533,7 +533,7 @@ tracing-appender = "0.2.5" url = { version = "2.3", default-features = false } zstd = "0.13" byteorder = "1" -fixed-cache = { version = "0.1.10", features = ["stats"] } +fixed-cache = { version = "0.1.10", default-features = false, features = ["alloc"] } moka = "0.12" tar-no-std = { version = "0.4.2", default-features = false } miniz_oxide = { version = "0.9.0", default-features = false } diff --git a/Dockerfile.depot b/Dockerfile.depot index 311ee8e0621..0ff3f2b736a 100644 --- a/Dockerfile.depot +++ b/Dockerfile.depot @@ -77,6 +77,8 @@ ARG BINARY=reth # Runtime defaults ARG RETH_ENGINE_TXPOOL_PREWARMING=false ENV RETH_ENGINE_TXPOOL_PREWARMING=$RETH_ENGINE_TXPOOL_PREWARMING +ARG RETH_ENGINE_SENDER_RECOVERY_CACHE=false +ENV RETH_ENGINE_SENDER_RECOVERY_CACHE=$RETH_ENGINE_SENDER_RECOVERY_CACHE # Install runtime dependencies RUN apt-get update && \ diff --git a/crates/ethereum/evm/src/lib.rs b/crates/ethereum/evm/src/lib.rs index 5fb989c3ea8..1cb5a7b1670 100644 --- a/crates/ethereum/evm/src/lib.rs +++ b/crates/ethereum/evm/src/lib.rs @@ -30,7 +30,7 @@ use reth_chainspec::{ChainSpec, EthChainSpec, MAINNET}; use reth_ethereum_primitives::{Block, EthPrimitives, TransactionSigned}; use reth_evm::{ eth::NextEvmEnvAttributes, precompiles::PrecompilesMap, ConfigureEvm, EvmEnv, EvmFactory, - JitBackend, NextBlockEnvAttributes, TransactionEnvMut, + JitBackend, NextBlockEnvAttributes, SenderRecoveryCache, TransactionEnvMut, }; use reth_primitives_traits::{SealedBlock, SealedHeader}; use revm::{context::BlockEnv, primitives::hardfork::SpecId}; @@ -87,6 +87,8 @@ pub struct EthEvmConfig { pub executor_factory: EthBlockExecutorFactory, EvmFactory>, /// Ethereum block assembler. pub block_assembler: EthBlockAssembler, + /// Cache of recovered transaction senders, if enabled. + pub sender_recovery_cache: Option, } impl EthEvmConfig { @@ -113,6 +115,7 @@ impl EthEvmConfig { pub fn new_with_evm_factory(chain_spec: Arc, evm_factory: EvmFactory) -> Self { Self { block_assembler: EthBlockAssembler::new(chain_spec.clone()), + sender_recovery_cache: None, executor_factory: EthBlockExecutorFactory::new( RethReceiptBuilder::default(), chain_spec, @@ -125,6 +128,12 @@ impl EthEvmConfig { pub const fn chain_spec(&self) -> &Arc { self.executor_factory.spec() } + + /// Uses the provided sender recovery cache. + pub fn with_sender_recovery_cache(mut self, cache: SenderRecoveryCache) -> Self { + self.sender_recovery_cache = Some(cache); + self + } } impl ConfigureEvm for EthEvmConfig @@ -345,10 +354,16 @@ where payload: &ExecutionData, ) -> Result, Self::Error> { let txs = payload.payload.transactions().clone(); - let convert = |tx: Bytes| { + let sender_recovery_cache = self.sender_recovery_cache.clone(); + let convert = move |tx: Bytes| { let tx = TxTy::::decode_2718_exact(tx.as_ref()).map_err(AnyError::new)?; - let signer = tx.try_recover().map_err(AnyError::new)?; + let signer = if let Some(cache) = &sender_recovery_cache { + cache.recover(&tx) + } else { + tx.try_recover() + } + .map_err(AnyError::new)?; Ok::<_, AnyError>(tx.with_signer(signer)) }; diff --git a/crates/ethereum/node/src/node.rs b/crates/ethereum/node/src/node.rs index 06ff516f6e2..0746e679198 100644 --- a/crates/ethereum/node/src/node.rs +++ b/crates/ethereum/node/src/node.rs @@ -607,7 +607,10 @@ where let jit = &ctx.config().jit; let dump_dir = jit.debug.then(|| ctx.config().datadir().data_dir().join("jit")); - let (evm_config, revmc_metrics) = build_evm_config(ctx.chain_spec(), jit, dump_dir)?; + let (mut evm_config, revmc_metrics) = build_evm_config(ctx.chain_spec(), jit, dump_dir)?; + if let Some(cache) = ctx.sender_recovery_cache() { + evm_config = evm_config.with_sender_recovery_cache(cache.clone()); + } #[cfg(not(feature = "jit"))] let _ = revmc_metrics; diff --git a/crates/evm/evm/Cargo.toml b/crates/evm/evm/Cargo.toml index ab367006f8e..4ab05984689 100644 --- a/crates/evm/evm/Cargo.toml +++ b/crates/evm/evm/Cargo.toml @@ -34,6 +34,7 @@ derive_more.workspace = true futures-util.workspace = true metrics = { workspace = true, optional = true } rayon = { workspace = true, optional = true } +fixed-cache.workspace = true [dev-dependencies] reth-ethereum-primitives.workspace = true @@ -57,6 +58,7 @@ std = [ "reth-trie-common/std", "reth-ethereum-primitives/std", "alloy-eip7928/std", + "fixed-cache/std", ] metrics = ["std", "dep:metrics", "dep:reth-metrics"] test-utils = [ diff --git a/crates/evm/evm/src/lib.rs b/crates/evm/evm/src/lib.rs index e9ef87ad4e3..4dfb0b6a841 100644 --- a/crates/evm/evm/src/lib.rs +++ b/crates/evm/evm/src/lib.rs @@ -44,6 +44,8 @@ pub use aliases::*; mod engine; #[cfg(feature = "std")] pub use engine::{ConfigureEngineEvm, ConvertTx, ExecutableTxIterator, ExecutableTxTuple}; +mod sender_recovery; +pub use sender_recovery::SenderRecoveryCache; #[cfg(feature = "metrics")] pub mod metrics; diff --git a/crates/evm/evm/src/sender_recovery.rs b/crates/evm/evm/src/sender_recovery.rs new file mode 100644 index 00000000000..de9d1f51280 --- /dev/null +++ b/crates/evm/evm/src/sender_recovery.rs @@ -0,0 +1,90 @@ +use alloc::sync::Arc; +use alloy_primitives::{map::FbBuildHasher, Address, B256}; +use reth_primitives_traits::{transaction::signed::RecoveryError, SignedTransaction}; + +/// Number of entries retained in the default sender recovery cache. +const SENDER_RECOVERY_CACHE_CAPACITY: usize = 1 << 17; + +/// Shared cache of recovered transaction senders. +/// +/// Sender recovery is performed when a transaction enters the pool and again when the same +/// transaction is received in an execution payload. Sharing this bounded, lock-free cache lets +/// payload prewarming reuse the result produced by transaction-pool ingress. +#[derive(Clone, Debug)] +pub struct SenderRecoveryCache { + cache: Arc, SenderRecoveryCacheConfig>>, +} + +impl SenderRecoveryCache { + /// Creates a sender recovery cache with the given capacity. + /// + /// The capacity must satisfy [`fixed_cache::Cache`] capacity requirements. + pub fn new(capacity: usize) -> Self { + Self { cache: Arc::new(fixed_cache::Cache::new(capacity, FbBuildHasher::<32>::default())) } + } + + /// Returns a cached sender for the transaction hash. + #[inline] + pub fn get(&self, tx_hash: &B256) -> Option

{ + self.cache.get(tx_hash) + } + + /// Returns the cached sender or recovers and caches it on a miss. + /// + /// Failed recoveries are not cached. + #[inline] + pub fn recover(&self, transaction: &T) -> Result { + self.cache.get_or_try_insert_with_ref( + transaction.tx_hash(), + |_| transaction.try_recover(), + |hash| *hash, + ) + } +} + +impl Default for SenderRecoveryCache { + fn default() -> Self { + Self::new(SENDER_RECOVERY_CACHE_CAPACITY) + } +} + +struct SenderRecoveryCacheConfig; + +impl fixed_cache::CacheConfig for SenderRecoveryCacheConfig { + const STATS: bool = false; +} + +#[cfg(test)] +mod tests { + use super::*; + use alloy_consensus::TxLegacy; + use alloy_primitives::{Signature, U256}; + use reth_ethereum_primitives::{Transaction, TransactionSigned}; + + #[test] + fn recover_populates_cache() { + let transaction = TransactionSigned::new_unhashed( + Transaction::Legacy(TxLegacy::default()), + Signature::test_signature(), + ); + let cache = SenderRecoveryCache::new(4); + let shared_cache = cache.clone(); + + let sender = cache.recover(&transaction).unwrap(); + + assert_eq!(shared_cache.get(transaction.tx_hash()), Some(sender)); + assert_eq!(shared_cache.recover(&transaction).unwrap(), sender); + } + + #[test] + fn failed_recovery_is_not_cached() { + let transaction = TransactionSigned::new_unhashed( + Transaction::Legacy(TxLegacy::default()), + Signature::new(U256::ZERO, U256::ZERO, false), + ); + let cache = SenderRecoveryCache::new(4); + + assert!(cache.recover(&transaction).is_err()); + assert_eq!(cache.get(transaction.tx_hash()), None); + } +} diff --git a/crates/net/network/Cargo.toml b/crates/net/network/Cargo.toml index 91a52bbf0f0..af752ef3f29 100644 --- a/crates/net/network/Cargo.toml +++ b/crates/net/network/Cargo.toml @@ -24,6 +24,7 @@ reth-discv4.workspace = true reth-discv5.workspace = true reth-dns-discovery.workspace = true reth-ethereum-forks.workspace = true +reth-evm.workspace = true reth-eth-wire.workspace = true reth-eth-wire-types.workspace = true reth-ecies.workspace = true @@ -150,4 +151,5 @@ test-utils = [ "reth-tasks/test-utils", "reth-stages-types/test-utils", "reth-trie/test-utils", + "reth-evm/test-utils", ] diff --git a/crates/net/network/src/builder.rs b/crates/net/network/src/builder.rs index 82309393a24..3ed5b92fac1 100644 --- a/crates/net/network/src/builder.rs +++ b/crates/net/network/src/builder.rs @@ -34,6 +34,15 @@ pub struct NetworkBuilder // === impl NetworkBuilder === impl NetworkBuilder { + /// Maps the transactions component. + pub fn map_transactions(self, f: F) -> NetworkBuilder + where + F: FnOnce(Tx) -> NewTx, + { + let Self { network, transactions, request_handler } = self; + NetworkBuilder { network, transactions: f(transactions), request_handler } + } + /// Consumes the type and returns all fields. pub fn split(self) -> (NetworkManager, Tx, Eth) { let Self { network, transactions, request_handler } = self; diff --git a/crates/net/network/src/transactions/mod.rs b/crates/net/network/src/transactions/mod.rs index be28944ccaf..206a8b5b80a 100644 --- a/crates/net/network/src/transactions/mod.rs +++ b/crates/net/network/src/transactions/mod.rs @@ -55,6 +55,7 @@ use reth_eth_wire::{ PooledTransactions, RequestTxHashes, Transactions, ValidAnnouncementData, }; use reth_ethereum_primitives::TxType; +use reth_evm::SenderRecoveryCache; use reth_metrics::common::mpsc::MemoryBoundedReceiver; use reth_network_api::{ events::{PeerEvent, SessionInfo}, @@ -285,6 +286,8 @@ impl TransactionsHandle { pub struct TransactionsManager { /// Access to the transaction pool. pool: Pool, + /// Cache of recovered transaction senders shared with payload execution, if enabled. + sender_recovery_cache: Option, /// Network access. network: NetworkHandle, /// Subscriptions to all network related events. @@ -400,6 +403,7 @@ impl TransactionsManager { Self { pool, + sender_recovery_cache: None, network, network_events, transaction_fetcher, @@ -424,6 +428,12 @@ impl TransactionsManager { TransactionsHandle { manager_tx: self.command_tx.clone() } } + /// Uses the provided sender recovery cache. + pub fn with_sender_recovery_cache(mut self, cache: SenderRecoveryCache) -> Self { + self.sender_recovery_cache = Some(cache); + self + } + /// Returns `true` if [`TransactionsManager`] has capacity to request pending hashes. Returns /// `false` if [`TransactionsManager`] is operating close to full capacity. fn has_capacity_for_fetching_pending_hashes(&self) -> bool { @@ -1454,16 +1464,23 @@ where let txs_len = transactions.len(); - let recover = |tx| match Pool::Transaction::try_recover(tx) { - Ok(tx) => Some(tx), - Err(badtx) => { - trace!(target: "net::tx", - peer_id=format!("{peer_id:#}"), - hash=%badtx.tx_hash(), - client_version=%client_version, - "failed ecrecovery for transaction" - ); - None + let recover = |tx| { + let recovered = if let Some(cache) = &self.sender_recovery_cache { + Pool::Transaction::try_recover_with_cache(tx, cache) + } else { + Pool::Transaction::try_recover(tx) + }; + match recovered { + Ok(tx) => Some(tx), + Err(badtx) => { + trace!(target: "net::tx", + peer_id=format!("{peer_id:#}"), + hash=%badtx.tx_hash(), + client_version=%client_version, + "failed ecrecovery for transaction" + ); + None + } } }; diff --git a/crates/node/builder/src/builder/mod.rs b/crates/node/builder/src/builder/mod.rs index 5660a7bf49a..d7952de9329 100644 --- a/crates/node/builder/src/builder/mod.rs +++ b/crates/node/builder/src/builder/mod.rs @@ -751,17 +751,24 @@ pub struct BuilderContext { pub(crate) executor: TaskExecutor, /// Config container pub(crate) config_container: WithConfigs<::ChainSpec>, + /// Cache of recovered transaction senders shared by node components, if enabled. + sender_recovery_cache: Option, } impl BuilderContext { /// Create a new instance of [`BuilderContext`] - pub const fn new( + pub fn new( head: Head, provider: Node::Provider, executor: TaskExecutor, config_container: WithConfigs<::ChainSpec>, ) -> Self { - Self { head, provider, executor, config_container } + let sender_recovery_cache = config_container + .config + .engine + .sender_recovery_cache_enabled + .then(reth_evm::SenderRecoveryCache::default); + Self { head, provider, executor, config_container, sender_recovery_cache } } /// Returns the configured provider to interact with the blockchain. @@ -796,6 +803,11 @@ impl BuilderContext { &self.executor } + /// Returns the sender recovery cache shared by node components, if enabled. + pub const fn sender_recovery_cache(&self) -> Option<&reth_evm::SenderRecoveryCache> { + self.sender_recovery_cache.as_ref() + } + /// Returns the chain spec of the node. pub fn chain_spec(&self) -> Arc<::ChainSpec> { self.provider().chain_spec() @@ -920,6 +932,13 @@ impl BuilderContext { propagation_policy, announcement_policy, ) + .map_transactions(|transactions| { + if let Some(cache) = self.sender_recovery_cache.clone() { + transactions.with_sender_recovery_cache(cache) + } else { + transactions + } + }) .request_handler_with_blob_store(self.provider().clone(), pool.blob_store()) .split_with_handle(); diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index 66d7e19b5ee..2d16bf1277c 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -32,6 +32,7 @@ pub struct DefaultEngineValues { state_cache_disabled: bool, prewarming_disabled: bool, txpool_prewarming_enabled: bool, + sender_recovery_cache_enabled: bool, state_provider_metrics: bool, cross_block_cache_size: usize, state_root_task_compare_updates: bool, @@ -109,6 +110,12 @@ impl DefaultEngineValues { self } + /// Set whether to enable sender recovery caching by default + pub const fn with_sender_recovery_cache_enabled(mut self, v: bool) -> Self { + self.sender_recovery_cache_enabled = v; + self + } + /// Set whether to enable state provider metrics by default pub const fn with_state_provider_metrics(mut self, v: bool) -> Self { self.state_provider_metrics = v; @@ -255,6 +262,7 @@ impl Default for DefaultEngineValues { state_cache_disabled: false, prewarming_disabled: false, txpool_prewarming_enabled: false, + sender_recovery_cache_enabled: false, state_provider_metrics: false, cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE_MB, state_root_task_compare_updates: false, @@ -353,6 +361,15 @@ pub struct EngineArgs { )] pub txpool_prewarming_enabled: bool, + /// Enable caching recovered transaction senders across transaction ingress and payload + /// execution. + #[arg( + long = "engine.sender-recovery-cache", + env = "RETH_ENGINE_SENDER_RECOVERY_CACHE", + default_value_t = DefaultEngineValues::get_global().sender_recovery_cache_enabled + )] + pub sender_recovery_cache_enabled: bool, + /// CAUTION: This CLI flag has no effect anymore. The parallel sparse trie is always enabled. #[deprecated] #[arg(long = "engine.parallel-sparse-trie", default_value = "true", hide = true)] @@ -551,6 +568,7 @@ impl Default for EngineArgs { state_cache_disabled, prewarming_disabled, txpool_prewarming_enabled, + sender_recovery_cache_enabled, state_provider_metrics, cross_block_cache_size, state_root_task_compare_updates, @@ -585,6 +603,7 @@ impl Default for EngineArgs { state_cache_disabled, prewarming_disabled, txpool_prewarming_enabled, + sender_recovery_cache_enabled, parallel_sparse_trie_enabled: true, parallel_sparse_trie_disabled: false, state_provider_metrics, @@ -807,6 +826,17 @@ mod tests { assert_eq!(args.persistence_backpressure_threshold(), 101); } + #[test] + fn sender_recovery_cache_is_disabled_by_default_and_can_be_enabled() { + let args = CommandParser::::parse_from(["reth"]).args; + assert!(!args.sender_recovery_cache_enabled); + + let args = + CommandParser::::parse_from(["reth", "--engine.sender-recovery-cache"]) + .args; + assert!(args.sender_recovery_cache_enabled); + } + #[test] #[allow(deprecated)] fn engine_args() { @@ -821,6 +851,7 @@ mod tests { prewarming_disabled: true, // conflicts with --engine.disable-state-cache, covered by its own test below txpool_prewarming_enabled: false, + sender_recovery_cache_enabled: true, parallel_sparse_trie_enabled: true, parallel_sparse_trie_disabled: false, state_provider_metrics: true, @@ -864,6 +895,7 @@ mod tests { "--engine.legacy-state-root", "--engine.disable-state-cache", "--engine.disable-prewarming", + "--engine.sender-recovery-cache", "--engine.state-provider-metrics", "--engine.cross-block-cache-size", "256", diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 877e1e6e6f1..2507d200819 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1387,6 +1387,17 @@ pub trait PoolTransaction: pooled.try_into_recovered().map(Self::from_pooled) } + /// Recovers and converts a pooled transaction using the provided sender recovery cache. + fn try_recover_with_cache( + pooled: Self::Pooled, + cache: &reth_evm::SenderRecoveryCache, + ) -> Result { + match cache.recover(&pooled) { + Ok(signer) => Ok(Self::from_pooled(Recovered::new_unchecked(pooled, signer))), + Err(_) => Err(pooled), + } + } + /// Decodes and recovers a raw transaction into this pool transaction type. /// /// Implementations can override this to avoid constructing the pooled transaction as an diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index b63deba860b..23d46efbdb7 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1016,6 +1016,11 @@ Engine: [env: RETH_ENGINE_TXPOOL_PREWARMING=] + --engine.sender-recovery-cache + Enable caching recovered transaction senders across transaction ingress and payload execution + + [env: RETH_ENGINE_SENDER_RECOVERY_CACHE=] + --engine.state-provider-metrics Enable state provider latency metrics. This allows the engine to collect and report stats about how long state provider calls took during execution, but this does introduce slight overhead to state provider calls From dff579b6464a6da18dad8ef2fbe4f83873346a22 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 28 Jul 2026 11:30:42 +0200 Subject: [PATCH 054/148] chore(deps): update Alloy to 2.2.0 (#26517) --- Cargo.lock | 124 ++++++++++++++++++++++---------------------- Cargo.toml | 54 +++++++++---------- bin/reth/Cargo.toml | 2 +- 3 files changed, 90 insertions(+), 90 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b96c4c7ba56..2c2f5155c7b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a6ff0c4adba2abdcd9fb5829ae5f4394c06f8585ed283a9ba79aa33763c802e1" +checksum = "1f0f42ee16e9eaffa4df5aa3a17b1df88dd222310d13152ab91357a2551fd054" dependencies = [ "alloy-eips", "alloy-primitives", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cdf48932b1db3216175e19a2b476d89d53076e004850ee7983c6807ba0fde74" +checksum = "db388e804269d977d60fa7530ccfefe9184ca48d4055dad3ec7e649a29186a06" dependencies = [ "alloy-consensus", "alloy-eips", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee5c8b5baa015ef1d07a33983794e1539cce36954846a9bc6e1050d8a1ebf9b7" +checksum = "69937abc65310c6eb739e540b107688943b61f38e7dcf9fde769367d3608d657" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea4c0453065b9206acc0f869a258dc8dcbbd595e144b4446f2c493a24a814d1f" +checksum = "52010e7c255f45b0ef3ca18144cbfbe0c044b055fe53f59323f3df5d95f265b1" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027ba57264c5d05e4ff52e6090b3592e7526f5d5f5a844add6bbdd17c071c911" +checksum = "2a60fa6337a6af8bae818de3030448827bf55c1321c90c7d1f6f3b79257ead2a" dependencies = [ "alloy-eips", "alloy-primitives", @@ -365,9 +365,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c4691c60de5d628533752cd07e102d17c47874c06c04c91af33960fd94c484f4" +checksum = "24d7ff4d7f1503d12edda9b241904a309d6e1d5bcee3f0ffdc4d3d3f16f366b8" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -380,9 +380,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d912bb639bf4ac31e83095afe9e907c8b9774ce0c405966228368309fcfc45f" +checksum = "44ff1439d901b44f6ff840545f82d65e66830ee33fb33cae495a566d66e0ccc8" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -406,9 +406,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d875a11bd98595f57f73890f2e36f4a1cca0fa670623e59c9fb08b2389979c36" +checksum = "fb920286c4a71fb0ff3bbb207a640a531a42fceef5b43d4a943f7d5265789876" dependencies = [ "alloy-consensus", "alloy-eips", @@ -419,9 +419,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ab58d45ceaabba867fc05f018f58e4640df2e6424b4c759d45f66cf075ad8fe" +checksum = "bd2d06c4044c3ae7f9b7a70c9a7bb3bde9a08e99f4d9aa91001e17a182b562dc" dependencies = [ "alloy-genesis", "alloy-hardforks 0.2.13", @@ -473,9 +473,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e7d526d184d392a8fbcf4293e457789b307bb70646e4f8b902989a246529450" +checksum = "329375f50202cc6b2dcc8c6b1e53b716cbbebbeea54e931befc4f3b628dfc469" dependencies = [ "alloy-chains", "alloy-consensus", @@ -517,9 +517,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa5976a77e32a63152e937fa90d0dae1f8142b41a9a99a6386d6ea58934cfa6" +checksum = "71044e076a7c243454619e36279fc4654bc1ff846fbf666663ef574388859c84" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -561,9 +561,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "755447dc13a04c6fa6db8dedb5d32ab5edfa4dd7aa34487ff192a3d873e0e8cf" +checksum = "52bffef2c5714f578f267bb959b47908b95728f1f3e2073a96d928f4441c148d" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -587,9 +587,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96deb317fe224e98a8ae17a7ed7ea63478867385bf50b7abd53642b24cfd811a" +checksum = "a64449cae1cf37c4907b10ccca1613b71809f9a1f3ad6e2b4cc14d08e2e4947f" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -600,9 +600,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3279cb55f7069c2fb1dbcd9ab2c6e79a02d63c92fd0b850addea0a3715d6b05f" +checksum = "980671fae7bff5ab0ae2d0bfe5c97e42cda103fece106f8d0e584a41c789fc7c" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -612,9 +612,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f89d27756bf3effdac25d07692724e840ce90ca1ff956a6b47dd2ae1612f597" +checksum = "261411a259d346a69078237e2997a7ce0faba006baf5b1937023316d5ef07bf4" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -624,9 +624,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "911a513723ef0b90c3b072f65eebf54d6ebc8b651d06734e252d024bd89b88ac" +checksum = "4fe7fa70c3af4b2de7f670b50d680700e0f979ce3a0ab7725958b51ffc3b16c2" dependencies = [ "alloy-consensus-any", "alloy-network-primitives", @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81133671b8da58169589aac996f3c4da23bbdee85b13ecee7098065f3eae718a" +checksum = "d22759d1d0e79082a5a06f75dc45bf55dfedbb4ac9890da1f179d85a5d2f32f7" dependencies = [ "alloy-eips", "alloy-primitives", @@ -659,9 +659,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e9c1f9ac81c56b2d96901201db7eb95c4e9fc3c21237a4690f8c652aa62089" +checksum = "7b918cbaadbdcffa4ed64840a5ecbde80a6476ab23adc8c5c8ae6feec081058c" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -672,9 +672,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b1f682c4881aa7000ac7cc86d7aeeb3b09836a77a90b329820140233651aeea" +checksum = "d30ff2e8f7d05433c1d36cf0f1ba045bf071a3d139f63923742fbdfb171dfe44" dependencies = [ "alloy-consensus", "alloy-eips", @@ -693,9 +693,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e1bd19904581ee2075b61faabab1a4cefa8b96d8f2c85343adeae8ecf615e2b" +checksum = "04f60a196df184facc9acbb61bd765a36cfb3c9d27eb6b43d5bd26340df1b96a" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -715,9 +715,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54c635f0c45a871b55c276653e3838d1687d86282eeaf3d83a1914e02563b3f2" +checksum = "b04a8f30a439c995d71ab999d56a828dcbe4d90b4f012419832779e60acf86af" dependencies = [ "alloy-consensus", "alloy-eips", @@ -730,9 +730,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "796729a4e1783904c6040f11a503c4d55ddb16f69dc199ad15e50c4ad039f371" +checksum = "a12cafc0ee2290122ec4e069bb0f2389a356f4b5414f2e82de8288049c2bc98a" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -744,9 +744,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8533c450895de79eb581875bfc317d1a239ae71aba79e20ee158fa153268f163" +checksum = "157eec8f8661e6e70e3749c0a4d918a8831f0b0fc152aed60c0de457faca420b" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -756,9 +756,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1e97b3e0b9f816b25083045dcfa69431bd059a078e828e4d82d296d1949b96c" +checksum = "07758cbde42596b66da52bb661d77e253c3f7f9b3f553faf07adbadeb6419502" dependencies = [ "alloy-primitives", "arbitrary", @@ -768,9 +768,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6913d06ccc0d9c6ab67e2f82e2fbe292706da1f91442f30cded2d04b56bf0d58" +checksum = "bc43a73bc1bf7044898e1b0dcefaba45d0cfac3e379bb5ca29cc3e3a04c61451" dependencies = [ "alloy-primitives", "async-trait", @@ -783,9 +783,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c880813cd26bd1ddf8c2236e31295896efd1fcc5cc761d8894ae894503aeea53" +checksum = "372ff9be2b61c2cd6d33cfcef9bd5865962c0628c932767f5154443aba2c9746" dependencies = [ "alloy-consensus", "alloy-network", @@ -872,9 +872,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "999adfe5c91035c6bf4c4210e0eb8d0caed79d76fbf5e1b70d78721f6097ac04" +checksum = "830c87b05870c80c8d2cfb9245feaff36468893f19fc124c7ff563cabda6386c" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -895,9 +895,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e79a2c1793afc61eed9ca0963da370a988b27d2bbfa67c78013e262d47424c4" +checksum = "23ecd298fb0e6a31bd90c72efba2728e3d7569f1c89a3719576fb0e4e40b0fc8" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -911,9 +911,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c24422d5159996688b0c094babf1969cb0197d453902da483711c92c1624fea2" +checksum = "e7b51470da19d0ed256bb3cf31042cceca3ce01714a9ca76624fc71a999f6a27" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -931,9 +931,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bea36621cd4e4f06c1243e556b32fdc9c4db7b9b09b72a95a87383c0ffcc625" +checksum = "e67177a75a6e8a44f39ccbbc64a3e3a70186ef401d7313285b56735d4285aa34" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "2.1.1" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "406bc1183f6843e0aba09f7b3365e828b597213d60793ba5cb41befc863e3a78" +checksum = "e50185b84799c8b5f016a0c9da74309fe28d1c70fbe590a4e8d9cae6df3eebb0" dependencies = [ "darling 0.23.0", "proc-macro2", diff --git a/Cargo.toml b/Cargo.toml index 9bf45de6693..708eac1249e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -447,33 +447,33 @@ alloy-trie = { version = "0.9.4", default-features = false } alloy-hardforks = "0.4.7" -alloy-consensus = { version = "2.1.1", default-features = false } -alloy-contract = { version = "2.1.1", default-features = false } -alloy-eips = { version = "2.1.1", default-features = false } -alloy-genesis = { version = "2.1.1", default-features = false } -alloy-json-rpc = { version = "2.1.1", default-features = false } -alloy-network = { version = "2.1.1", default-features = false } -alloy-network-primitives = { version = "2.1.1", default-features = false } -alloy-provider = { version = "2.1.1", features = ["reqwest", "debug-api"], default-features = false } -alloy-pubsub = { version = "2.1.1", default-features = false } -alloy-rpc-client = { version = "2.1.1", default-features = false } -alloy-rpc-types = { version = "2.1.1", features = ["eth"], default-features = false } -alloy-rpc-types-admin = { version = "2.1.1", default-features = false } -alloy-rpc-types-anvil = { version = "2.1.1", default-features = false } -alloy-rpc-types-beacon = { version = "2.1.1", default-features = false } -alloy-rpc-types-debug = { version = "2.1.1", default-features = false } -alloy-rpc-types-engine = { version = "2.1.1", default-features = false } -alloy-rpc-types-eth = { version = "2.1.1", default-features = false } -alloy-rpc-types-mev = { version = "2.1.1", default-features = false } -alloy-rpc-types-trace = { version = "2.1.1", default-features = false } -alloy-rpc-types-txpool = { version = "2.1.1", default-features = false } -alloy-serde = { version = "2.1.1", default-features = false } -alloy-signer = { version = "2.1.1", default-features = false } -alloy-signer-local = { version = "2.1.1", default-features = false } -alloy-transport = { version = "2.1.1" } -alloy-transport-http = { version = "2.1.1", features = ["reqwest-rustls-tls"], default-features = false } -alloy-transport-ipc = { version = "2.1.1", default-features = false } -alloy-transport-ws = { version = "2.1.1", default-features = false } +alloy-consensus = { version = "2.2.0", default-features = false } +alloy-contract = { version = "2.2.0", default-features = false } +alloy-eips = { version = "2.2.0", default-features = false } +alloy-genesis = { version = "2.2.0", default-features = false } +alloy-json-rpc = { version = "2.2.0", default-features = false } +alloy-network = { version = "2.2.0", default-features = false } +alloy-network-primitives = { version = "2.2.0", default-features = false } +alloy-provider = { version = "2.2.0", features = ["reqwest", "debug-api"], default-features = false } +alloy-pubsub = { version = "2.2.0", default-features = false } +alloy-rpc-client = { version = "2.2.0", default-features = false } +alloy-rpc-types = { version = "2.2.0", features = ["eth"], default-features = false } +alloy-rpc-types-admin = { version = "2.2.0", default-features = false } +alloy-rpc-types-anvil = { version = "2.2.0", default-features = false } +alloy-rpc-types-beacon = { version = "2.2.0", default-features = false } +alloy-rpc-types-debug = { version = "2.2.0", default-features = false } +alloy-rpc-types-engine = { version = "2.2.0", default-features = false } +alloy-rpc-types-eth = { version = "2.2.0", default-features = false } +alloy-rpc-types-mev = { version = "2.2.0", default-features = false } +alloy-rpc-types-trace = { version = "2.2.0", default-features = false } +alloy-rpc-types-txpool = { version = "2.2.0", default-features = false } +alloy-serde = { version = "2.2.0", default-features = false } +alloy-signer = { version = "2.2.0", default-features = false } +alloy-signer-local = { version = "2.2.0", default-features = false } +alloy-transport = { version = "2.2.0" } +alloy-transport-http = { version = "2.2.0", features = ["reqwest-rustls-tls"], default-features = false } +alloy-transport-ipc = { version = "2.2.0", default-features = false } +alloy-transport-ws = { version = "2.2.0", default-features = false } # misc either = { version = "1.16.0", default-features = false } diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 2dc5d54b95f..0449dc71299 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -71,7 +71,7 @@ aquamarine.workspace = true clap = { workspace = true, features = ["derive", "env"] } [dev-dependencies] -alloy-node-bindings = "2.1.1" +alloy-node-bindings = "2.2.0" alloy-provider = { workspace = true, features = ["reqwest"] } alloy-rpc-types-eth.workspace = true backon.workspace = true From 4f72933a1f4653203c3aa5dca5eeddca427cf609 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 28 Jul 2026 11:31:13 +0200 Subject: [PATCH 055/148] fix: add missing dev-dependency features for standalone test builds (#26511) --- crates/engine/primitives/Cargo.toml | 3 +++ crates/etl/Cargo.toml | 2 +- crates/payload/util/Cargo.toml | 4 ++++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/crates/engine/primitives/Cargo.toml b/crates/engine/primitives/Cargo.toml index 11394aa8083..e73046c027f 100644 --- a/crates/engine/primitives/Cargo.toml +++ b/crates/engine/primitives/Cargo.toml @@ -39,6 +39,9 @@ auto_impl.workspace = true serde.workspace = true thiserror.workspace = true +[dev-dependencies] +alloy-primitives = { workspace = true, features = ["getrandom"] } + [features] default = ["std"] trie-debug = [] diff --git a/crates/etl/Cargo.toml b/crates/etl/Cargo.toml index e90f706f5c1..8ec5db8f7ac 100644 --- a/crates/etl/Cargo.toml +++ b/crates/etl/Cargo.toml @@ -16,4 +16,4 @@ reth-db-api.workspace = true rayon.workspace = true [dev-dependencies] -alloy-primitives.workspace = true +alloy-primitives = { workspace = true, features = ["getrandom"] } diff --git a/crates/payload/util/Cargo.toml b/crates/payload/util/Cargo.toml index f52484f92fe..b7aeba50058 100644 --- a/crates/payload/util/Cargo.toml +++ b/crates/payload/util/Cargo.toml @@ -18,3 +18,7 @@ reth-transaction-pool.workspace = true # alloy alloy-primitives.workspace = true alloy-consensus.workspace = true + +[dev-dependencies] +reth-transaction-pool = { workspace = true, features = ["test-utils"] } +alloy-primitives = { workspace = true, features = ["getrandom"] } From b8a214cbdd587f372fb47f01c621d3c565872910 Mon Sep 17 00:00:00 2001 From: Felix-Ayush <67006255+Ayush7614@users.noreply.github.com> Date: Tue, 28 Jul 2026 15:39:49 +0530 Subject: [PATCH 056/148] fix(net): reply to GetNodeData with empty payload (#26513) --- crates/net/network/src/eth_requests.rs | 36 ++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 853dbb1c51c..9254314202b 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -231,6 +231,21 @@ where let _ = response.send(Ok(BlockBodies(bodies))); } + /// Replies to `GetNodeData`. + /// + /// State serving via eth `GetNodeData` was removed; answer with an empty payload so eth/66 + /// peers receive a response instead of hanging until the request timeout when the oneshot is + /// dropped unanswered. + fn on_node_data_request( + &self, + _peer_id: PeerId, + _request: GetNodeData, + response: oneshot::Sender>, + ) { + self.metrics.eth_node_data_requests_received_total.increment(1); + let _ = response.send(Ok(NodeData(vec![]))); + } + fn on_receipts_request( &self, _peer_id: PeerId, @@ -691,8 +706,8 @@ where IncomingEthRequest::GetBlockBodies { peer_id, request, response } => { this.on_bodies_request(peer_id, request, response) } - IncomingEthRequest::GetNodeData { .. } => { - this.metrics.eth_node_data_requests_received_total.increment(1); + IncomingEthRequest::GetNodeData { peer_id, request, response } => { + this.on_node_data_request(peer_id, request, response) } IncomingEthRequest::GetReceipts { peer_id, request, response } => { this.on_receipts_request(peer_id, request, response) @@ -990,6 +1005,23 @@ mod tests { assert_eq!(get_cells_calls.load(Ordering::Relaxed), MAX_CELLS_SERVE); } + #[tokio::test] + async fn get_node_data_responds_with_empty_payload() { + let (peers_tx, _) = mpsc::unbounded_channel(); + let (_incoming_tx, incoming_rx) = mpsc::channel(1); + let handler = EthRequestHandler::::new( + NoopProvider::default(), + PeersHandle::new(peers_tx), + incoming_rx, + ); + let (response, rx) = oneshot::channel(); + + handler.on_node_data_request(PeerId::default(), GetNodeData(vec![B256::ZERO]), response); + + let node_data = rx.await.expect("response channel must not be dropped").unwrap(); + assert!(node_data.0.is_empty(), "GetNodeData should reply with empty NodeData"); + } + /// Creates a request handler backed by the mock provider for snap response tests. fn snap_handler( provider: MockEthProvider, From 839b84027f35273dbf4d7dfa493366e3d5686d19 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 28 Jul 2026 13:08:34 +0200 Subject: [PATCH 057/148] fix(refactor): skip transient selfdestruct storage wipes (#26433) --- Cargo.lock | 1 + .../stages/stages/src/stages/execution/mod.rs | 70 +++++- crates/stages/stages/tests/preimage.rs | 203 +++++++++++++++++- crates/trie/common/src/hashed_state.rs | 85 +++++++- crates/trie/parallel/src/state_root_task.rs | 49 ++++- crates/trie/trie/Cargo.toml | 4 + crates/trie/trie/src/hashed_cursor/mod.rs | 37 +++- testing/ef-tests/src/cases/blockchain_test.rs | 14 +- 8 files changed, 443 insertions(+), 20 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2c2f5155c7b..3ecccc2b1d8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10549,6 +10549,7 @@ dependencies = [ "reth-tracing", "reth-trie-common", "reth-trie-sparse", + "revm", "tracing", "triehash", ] diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index 3376823550a..1ba0e71fd87 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -23,7 +23,8 @@ use reth_stages_api::{ UnwindInput, UnwindOutput, }; use reth_static_file_types::StaticFileSegment; -use reth_trie::KeccakKeyHasher; +use reth_trie::{hashed_cursor::zero_destroyed_account_storage, KeccakKeyHasher}; +use reth_trie_db::DatabaseHashedCursorFactory; use std::{ cmp::{max, Ordering}, collections::BTreeMap, @@ -497,7 +498,12 @@ where provider.write_state(&state, OriginalValuesKnown::Yes, StateWriteConfig::default())?; if provider.cached_storage_settings().use_hashed_state() { - let hashed_state = state.hash_state_slow::(); + let mut hashed_state = state.hash_state_slow::(); + zero_destroyed_account_storage( + &DatabaseHashedCursorFactory::new(provider.tx_ref()), + state.bundle.state(), + &mut hashed_state, + )?; provider.write_hashed_state(&hashed_state.into_sorted())?; } @@ -768,8 +774,10 @@ mod tests { }; use reth_prune::PruneModes; use reth_prune_types::{PruneMode, ReceiptsLogPruneConfig}; + use reth_revm::revm::database::{AccountStatus, BundleAccount}; use reth_stages_api::StageUnitCheckpoint; use reth_testing_utils::generators; + use reth_trie::HashedPostState; use std::collections::BTreeMap; fn stage() -> ExecutionStage { @@ -792,6 +800,64 @@ mod tests { ) } + #[test] + fn destroyed_storage_is_materialized_without_reverts() { + let factory = create_test_provider_factory(); + let provider = factory.database_provider_rw().unwrap(); + let address = Address::repeat_byte(0x11); + let hashed_address = keccak256(address); + let retained_slot = B256::repeat_byte(0x22); + let deleted_slot = B256::repeat_byte(0x33); + let retained_value = U256::from(1); + + provider + .tx_ref() + .put::( + hashed_address, + StorageEntry { key: retained_slot, value: U256::from(2) }, + ) + .unwrap(); + provider + .tx_ref() + .put::( + hashed_address, + StorageEntry { key: deleted_slot, value: U256::from(3) }, + ) + .unwrap(); + + let mut state = ExecutionOutcome::<()>::default(); + state.bundle.state.insert( + address, + BundleAccount::new( + Some(Default::default()), + Some(Default::default()), + Default::default(), + AccountStatus::DestroyedChanged, + ), + ); + + let mut hashed_state = HashedPostState::default(); + hashed_state + .storages + .entry(hashed_address) + .or_default() + .storage + .insert(retained_slot, retained_value); + + zero_destroyed_account_storage( + &DatabaseHashedCursorFactory::new(provider.tx_ref()), + state.bundle.state(), + &mut hashed_state, + ) + .unwrap(); + + let storage = &hashed_state.storages[&hashed_address]; + assert!(!storage.wiped); + assert_eq!(storage.storage[&retained_slot], retained_value); + assert_eq!(storage.storage[&deleted_slot], U256::ZERO); + assert!(state.bundle.reverts.is_empty()); + } + #[test] fn execution_checkpoint_matches() { let factory = create_test_provider_factory(); diff --git a/crates/stages/stages/tests/preimage.rs b/crates/stages/stages/tests/preimage.rs index de64709f036..ddcb1f74808 100644 --- a/crates/stages/stages/tests/preimage.rs +++ b/crates/stages/stages/tests/preimage.rs @@ -12,6 +12,11 @@ use reth_chainspec::{ }; use reth_config::config::StageConfig; use reth_consensus::noop::NoopConsensus; +use reth_db::tables; +use reth_db_api::{ + cursor::{DbCursorRO, DbDupCursorRO}, + transaction::DbTx, +}; use reth_db_common::init::{init_genesis, init_genesis_with_settings}; use reth_downloaders::{ bodies::bodies::BodiesDownloaderBuilder, file_client::FileClient, @@ -44,8 +49,10 @@ use reth_stages_api::{Pipeline, StageSet}; use reth_static_file::StaticFileProducer; use reth_storage_api::{StorageChangeSetReader, StorageSettings, StorageSettingsCache}; use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair}; -use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot}; -use reth_trie_db::DatabaseStateRoot; +use reth_trie::{ + hashed_cursor::zero_destroyed_account_storage, HashedPostState, KeccakKeyHasher, StateRoot, +}; +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; use std::{collections::BTreeMap, path::Path, sync::Arc}; use tokio::sync::watch; @@ -57,17 +64,18 @@ const TEST_CREATE2_SALT: B256 = B256::with_last_byte(0x42); /// Scenario coverage: /// 1. Cross-batch pre-Cancun wipe writes plain slot keys into storage changesets. -/// 2. Preimage DB exists and is usable across multiple stage executions. -/// 3. Post-Cancun execution removes the preimage DB directory. +/// 2. The destroyed account's hashed storage is removed. +/// 3. Preimage DB exists and is usable across multiple stage executions. +/// 4. Post-Cancun execution removes the preimage DB directory. /// /// Verifies v2 selfdestruct handling across a pre-/post-Cancun boundary. /// /// Test flow: /// 1. Run block 1 (pre-Cancun) and assert the `preimage/` MDBX directory exists and contains /// `keccak(slot) -> slot` rows for the two written storage slots. -/// 2. Run block 2 (pre-Cancun selfdestruct) and assert storage changesets for the destroyed account -/// contain exactly those two slots as **plain** keys with the expected prior values (`0x2a`, -/// `0x99`). +/// 2. Run block 2 (pre-Cancun selfdestruct) and assert the account's hashed storage is empty and +/// its storage changesets contain exactly those two slots as **plain** keys with the expected +/// prior values (`0x2a`, `0x99`). /// 3. Run block 3 (post-Cancun) and assert `preimage/` is removed, since this auxiliary DB is no /// longer needed after Cancun semantics are active. #[tokio::test(flavor = "multi_thread")] @@ -100,6 +108,14 @@ async fn test_pipeline_v2_selfdestruct_changesets_use_plain_slots() -> eyre::Res let expected_slots = scenario.expected_slots; assert!(preimage_path.exists(), "preimage dir should exist after first pre-Cancun run"); assert_preimage_rows(&preimage_path, &expected_slots)?; + assert_hashed_storage_entries( + &provider, + scenario.selfdestruct_contract, + &[ + (B256::with_last_byte(0x01), U256::from(0x2a)), + (B256::with_last_byte(0x02), U256::from(0x99)), + ], + )?; let local_head = pipeline_provider_factory.sealed_header(1)?.expect("block 1 header should exist"); @@ -121,6 +137,7 @@ async fn test_pipeline_v2_selfdestruct_changesets_use_plain_slots() -> eyre::Res assert!(preimage_path.exists(), "preimage dir should still exist after second pre-Cancun run"); assert_preimage_rows(&preimage_path, &expected_slots)?; assert_destroyed_changeset_entries(&provider, scenario.selfdestruct_contract)?; + assert_hashed_storage_entries(&provider, scenario.selfdestruct_contract, &[])?; let third_local_head = pipeline_provider_factory.sealed_header(2)?.expect("block 2 header should exist"); @@ -141,6 +158,41 @@ async fn test_pipeline_v2_selfdestruct_changesets_use_plain_slots() -> eyre::Res Ok(()) } +/// Prefunding ensures the CREATE2 target has original account state, while its init code performs +/// no `SSTORE`. This verifies that destruction alone does not create an otherwise-empty +/// [`reth_trie::HashedStorage`] solely to carry `wiped = true`. +#[tokio::test(flavor = "multi_thread")] +async fn test_pipeline_v2_prefunded_create2_selfdestruct_does_not_wipe_storage() -> eyre::Result<()> +{ + reth_tracing::init_test_tracing(); + + let scenario = setup_create2_selfdestruct_scenario()?; + let hashed_address = keccak256(scenario.child_contract); + + assert!(scenario.child_was_destroyed); + let storage = scenario.hashed_state.storages.get(&hashed_address); + assert!(storage.is_none()); + + let (pipeline_provider_factory, pipeline_genesis) = + init_v2_pipeline_provider_factory(scenario.chain_spec)?; + run_pipeline_range( + pipeline_provider_factory.clone(), + create_file_client_from_blocks(vec![scenario.block]), + pipeline_genesis, + 1..=1, + 1, + ) + .await?; + + let provider = pipeline_provider_factory.provider()?; + assert_eq!(provider.last_block_number()?, 1, "pipeline should sync block 1"); + assert!(provider.tx_ref().get::(hashed_address)?.is_none()); + assert_hashed_storage_entries(&provider, scenario.child_contract, &[])?; + assert_destroyed_changeset_entries_in_block(&provider, 1, scenario.child_contract, &[])?; + + Ok(()) +} + /// Scenario coverage: /// 1. Single execution batch (`1..=2`) where block 1 writes and block 2 wipes. /// 2. Block-by-block storage changesets still contain plain keys for wipe entries. @@ -340,6 +392,14 @@ struct SelfdestructScenario { expected_slots: [B256; 2], } +struct Create2SelfdestructScenario { + chain_spec: Arc, + block: SealedBlock, + child_contract: Address, + child_was_destroyed: bool, + hashed_state: HashedPostState, +} + fn setup_selfdestruct_scenario() -> eyre::Result { let mut rng = generators::rng(); let key_pair = generate_key(&mut rng); @@ -429,6 +489,95 @@ fn setup_selfdestruct_scenario() -> eyre::Result { }) } +fn setup_create2_selfdestruct_scenario() -> eyre::Result { + let mut rng = generators::rng(); + let key_pair = generate_key(&mut rng); + let signer_address = public_key_to_address(key_pair.public_key()); + let factory_contract = Address::new([0xaa; 20]); + let child_init = CREATE2_SELFDESTRUCT_INIT_CODE; + let child_contract = create2_address(factory_contract, TEST_CREATE2_SALT, &child_init); + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(Genesis { + alloc: [ + ( + signer_address, + GenesisAccount { + balance: U256::from(ETH_TO_WEI) * U256::from(1000), + ..Default::default() + }, + ), + ( + factory_contract, + GenesisAccount { + code: Some(CREATE2_FACTORY_RUNTIME_CODE), + ..Default::default() + }, + ), + (child_contract, GenesisAccount { balance: U256::ONE, ..Default::default() }), + ] + .into(), + ..MAINNET.genesis.clone() + }) + .shanghai_activated() + .with_fork(EthereumHardfork::Cancun, ForkCondition::Timestamp(30)) + .build(), + ); + + let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone()); + init_genesis(&provider_factory)?; + let genesis = provider_factory.sealed_header(0)?.expect("genesis should exist"); + let evm_config = EthEvmConfig::new(chain_spec.clone()); + let tx = sign_tx_with_key_pair( + key_pair, + Transaction::Eip1559(TxEip1559 { + chain_id: chain_spec.chain.id(), + nonce: 0, + gas_limit: 400_000, + max_fee_per_gas: INITIAL_BASE_FEE as u128, + max_priority_fee_per_gas: 0, + to: TxKind::Call(factory_contract), + input: child_init, + ..Default::default() + }), + ); + + let preview_block = RecoveredBlock::new_unhashed( + Block::new( + build_execution_header(genesis.hash(), 1, 12), + BlockBody { transactions: vec![tx.clone()], ommers: Vec::new(), withdrawals: None }, + ), + vec![signer_address], + ); + let output = { + let provider = provider_factory.database_provider_rw()?; + let state_provider = provider.latest(); + let db = StateProviderDatabase::new(&*state_provider); + evm_config.batch_executor(db).execute(&preview_block)? + }; + let child_was_destroyed = + output.state.account(&child_contract).is_some_and(|account| account.was_destroyed()); + let hashed_state = HashedPostState::from_bundle_state::(output.state.state()); + let block = execute_and_commit_block( + &provider_factory, + &evm_config, + signer_address, + genesis.hash(), + 1, + 12, + vec![tx], + )?; + + Ok(Create2SelfdestructScenario { + chain_spec, + block, + child_contract, + child_was_destroyed, + hashed_state, + }) +} + struct RevertedSlotSelfdestructScenario { chain_spec: Arc, blocks: Vec>, @@ -952,7 +1101,13 @@ fn execute_and_commit_block( }; let gas_used = output.gas_used; - let hashed_state = HashedPostState::from_bundle_state::(output.state.state()); + let mut hashed_state = + HashedPostState::from_bundle_state::(output.state.state()); + zero_destroyed_account_storage( + &DatabaseHashedCursorFactory::new(provider.tx_ref()), + output.state.state(), + &mut hashed_state, + )?; type TestStateRoot<'a, TX, A> = StateRoot< reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>, reth_trie_db::DatabaseHashedCursorFactory<&'a TX>, @@ -1101,6 +1256,32 @@ where Ok(()) } +fn assert_hashed_storage_entries

( + provider: &P, + address: Address, + expected: &[(B256, U256)], +) -> eyre::Result<()> +where + P: DBProvider, +{ + let mut cursor = provider.tx_ref().cursor_dup_read::()?; + let mut actual = Vec::new(); + if let Some((_, entry)) = cursor.seek_exact(keccak256(address))? { + actual.push((entry.key, entry.value)); + while let Some(entry) = cursor.next_dup_val()? { + actual.push((entry.key, entry.value)); + } + } + actual.sort_unstable_by_key(|(slot, _)| *slot); + + let mut expected = + expected.iter().map(|(slot, value)| (keccak256(*slot), *value)).collect::>(); + expected.sort_unstable_by_key(|(slot, _)| *slot); + + assert_eq!(actual, expected, "unexpected hashed storage for account {address}"); + Ok(()) +} + fn create_file_client_from_blocks(blocks: Vec>) -> Arc> { Arc::new(FileClient::from_blocks(blocks)) } @@ -1249,6 +1430,12 @@ const WRITE_RESTORE_THEN_SELFDESTRUCT_RUNTIME_CODE: Bytes = bytes!( "ff00" // SELFDESTRUCT; STOP ); +/// CREATE2 init code that selfdestructs before returning runtime bytecode. +const CREATE2_SELFDESTRUCT_INIT_CODE: Bytes = bytes!( + "737777777777777777777777777777777777777777" // PUSH20 beneficiary + "ff" // SELFDESTRUCT +); + /// Converts contract runtime bytecode into init code that returns the runtime. fn init_code_for_runtime(runtime: &Bytes) -> Bytes { let len = u8::try_from(runtime.len()).expect("runtime too large for PUSH1 init-code helper"); diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index 9bf04e5bb63..454bf947e7b 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -54,9 +54,12 @@ impl HashedPostState { .map(|(address, account)| { let hashed_address = KH::hash_key(address); let hashed_account = account.info.as_ref().map(Into::into); - let hashed_storage = HashedStorage::from_plain_storage( - account.status, - account.storage.iter().map(|(slot, value)| (slot, &value.present_value)), + let hashed_storage = HashedStorage::from_iter( + false, + account + .storage + .iter() + .map(|(slot, value)| (keccak256(B256::from(*slot)), value.present_value)), ); ( @@ -911,6 +914,17 @@ mod tests { state::{AccountInfo, Bytecode}, }; + fn bundle_hashed_storage(account: &BundleAccount) -> Option { + let address = Address::ZERO; + let mut state = + HashedPostState::from_bundle_state::([(&address, account)]); + state.storages.remove(&keccak256(address)) + } + + fn changed_storage(original: U256, present: U256) -> StorageWithOriginalValues { + core::iter::once((U256::from(1), StorageSlot::new_changed(original, present))).collect() + } + #[test] fn hashed_state_wiped_extension() { let hashed_address = B256::default(); @@ -1030,6 +1044,71 @@ mod tests { ); } + #[test] + fn destroyed_prefunded_account_without_storage_emits_no_storage() { + let original_info = AccountInfo { balance: U256::from(1), ..Default::default() }; + let account = BundleAccount::new( + Some(original_info), + None, + StorageWithOriginalValues::default(), + AccountStatus::Destroyed, + ); + + assert!(bundle_hashed_storage(&account).is_none()); + } + + #[test] + fn destroyed_accounts_emit_zero_storage_changes_without_wipe() { + let existing_contract = + AccountInfo { code_hash: B256::repeat_byte(0x01), ..Default::default() }; + let legacy_empty_account = AccountInfo::default(); + let prefunded_account = AccountInfo { balance: U256::from(1), ..Default::default() }; + + for original_info in + [Some(existing_contract), Some(legacy_empty_account), Some(prefunded_account), None] + { + let account = BundleAccount::new( + original_info, + None, + changed_storage(U256::from(2), U256::ZERO), + AccountStatus::Destroyed, + ); + + let storage = bundle_hashed_storage(&account).unwrap(); + let hashed_slot = keccak256(B256::from(U256::from(1))); + assert!(!storage.wiped); + assert_eq!(storage.storage[&hashed_slot], U256::ZERO); + } + } + + #[test] + fn destroyed_recreated_accounts_preserve_storage_without_wipe() { + let value = U256::from(2); + let new_account = BundleAccount::new( + None, + Some(AccountInfo::default()), + changed_storage(U256::ZERO, value), + AccountStatus::DestroyedChanged, + ); + let original_info = + AccountInfo { code_hash: B256::repeat_byte(0x01), ..Default::default() }; + let existing_account = BundleAccount::new( + Some(original_info), + Some(AccountInfo::default()), + changed_storage(U256::ZERO, value), + AccountStatus::DestroyedChanged, + ); + + let new_storage = bundle_hashed_storage(&new_account).unwrap(); + let existing_storage = bundle_hashed_storage(&existing_account).unwrap(); + let hashed_slot = keccak256(B256::from(U256::from(1))); + + assert!(!new_storage.wiped); + assert!(!existing_storage.wiped); + assert_eq!(new_storage.storage[&hashed_slot], value); + assert_eq!(existing_storage.storage[&hashed_slot], value); + } + #[test] fn test_hashed_post_state_with_accounts() { // Prepare random addresses and mock account info. diff --git a/crates/trie/parallel/src/state_root_task.rs b/crates/trie/parallel/src/state_root_task.rs index 9f1321d8563..d3a305a6cd3 100644 --- a/crates/trie/parallel/src/state_root_task.rs +++ b/crates/trie/parallel/src/state_root_task.rs @@ -513,9 +513,7 @@ pub fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostState { .map(|(slot, value)| (keccak256(B256::from(slot)), value.present_value)) .peekable(); - if destroyed { - hashed_state.storages.insert(hashed_address, HashedStorage::new(true)); - } else if changed_storage_iter.peek().is_some() { + if !destroyed && changed_storage_iter.peek().is_some() { hashed_state .storages .insert(hashed_address, HashedStorage::from_iter(false, changed_storage_iter)); @@ -529,11 +527,56 @@ pub fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostState { #[cfg(test)] mod tests { use super::*; + use alloy_primitives::{Address, U256}; + use revm::state::{Account, EvmStorageSlot, TransactionId}; use std::{ sync::atomic::{AtomicUsize, Ordering}, time::Duration, }; + #[test] + fn created_selfdestruct_does_not_emit_storage() { + let address = Address::repeat_byte(0x01); + let mut account = Account::default(); + account.mark_touch(); + assert!(account.mark_created_locally()); + assert!(account.mark_selfdestructed_locally()); + account.info.nonce = 1; + account.storage.insert( + U256::from(1), + EvmStorageSlot::new_changed(U256::ZERO, U256::from(2), TransactionId::ZERO), + ); + + let hashed_state = + evm_state_to_hashed_post_state(EvmState::from_iter([(address, account)])); + let hashed_address = keccak256(address); + + assert_eq!(hashed_state.accounts.get(&hashed_address), Some(&None)); + assert!(!hashed_state.storages.contains_key(&hashed_address)); + } + + #[test] + fn existing_selfdestruct_does_not_emit_storage() { + let address = Address::repeat_byte(0x02); + let mut account = Account::default(); + account.info.nonce = 1; + account.set_current_info_as_original(); + account.mark_touch(); + assert!(account.mark_selfdestructed_locally()); + account.selfdestruct(); + account.storage.insert( + U256::from(1), + EvmStorageSlot::new_changed(U256::ZERO, U256::from(2), TransactionId::ZERO), + ); + + let hashed_state = + evm_state_to_hashed_post_state(EvmState::from_iter([(address, account)])); + let hashed_address = keccak256(address); + + assert_eq!(hashed_state.accounts.get(&hashed_address), Some(&None)); + assert!(!hashed_state.storages.contains_key(&hashed_address)); + } + #[derive(Default)] struct CountingSink { access_hints: AtomicUsize, diff --git a/crates/trie/trie/Cargo.toml b/crates/trie/trie/Cargo.toml index 800c6417489..9b698d9a95c 100644 --- a/crates/trie/trie/Cargo.toml +++ b/crates/trie/trie/Cargo.toml @@ -27,6 +27,9 @@ alloy-primitives.workspace = true alloy-consensus.workspace = true alloy-trie.workspace = true +# revm +revm.workspace = true + # tracing tracing = { workspace = true, features = ["attributes"] } @@ -77,6 +80,7 @@ serde = [ "reth-stages-types/serde", "parking_lot/serde", "rand/serde", + "revm/serde", ] test-utils = [ "triehash", diff --git a/crates/trie/trie/src/hashed_cursor/mod.rs b/crates/trie/trie/src/hashed_cursor/mod.rs index 3b9d29964f8..3e5a0452975 100644 --- a/crates/trie/trie/src/hashed_cursor/mod.rs +++ b/crates/trie/trie/src/hashed_cursor/mod.rs @@ -1,6 +1,8 @@ -use alloy_primitives::{B256, U256}; +use alloy_primitives::{keccak256, Address, B256, U256}; use reth_primitives_traits::Account; use reth_storage_errors::db::DatabaseError; +use reth_trie_common::HashedPostState; +use revm::database::BundleAccount; /// Implementation of hashed state cursor traits for the post state. mod post_state; @@ -75,3 +77,36 @@ pub trait HashedStorageCursor: HashedCursor { /// After calling this method, the subsequent operation MUST be a [`HashedCursor::seek`] call. fn set_hashed_address(&mut self, hashed_address: B256); } + +/// Materializes storage deletions for destroyed accounts as explicit zero-valued slot updates. +/// +/// Final bundle values take precedence so that destroy-then-recreate transitions retain storage +/// written by the recreated account. +pub fn zero_destroyed_account_storage<'a>( + cursor_factory: &impl HashedCursorFactory, + accounts: impl IntoIterator, + hashed_state: &mut HashedPostState, +) -> Result<(), DatabaseError> { + let mut destroyed_accounts = accounts + .into_iter() + .filter(|(_, account)| account.was_destroyed()) + .map(|(address, _)| keccak256(address)); + let Some(mut hashed_address) = destroyed_accounts.next() else { return Ok(()) }; + let mut cursor = cursor_factory.hashed_storage_cursor(hashed_address)?; + + loop { + if let Some((hashed_slot, _)) = cursor.seek(B256::ZERO)? { + let storage = &mut hashed_state.storages.entry(hashed_address).or_default().storage; + storage.entry(hashed_slot).or_insert(U256::ZERO); + while let Some((hashed_slot, _)) = cursor.next()? { + storage.entry(hashed_slot).or_insert(U256::ZERO); + } + } + + let Some(next_hashed_address) = destroyed_accounts.next() else { break }; + hashed_address = next_hashed_address; + cursor.set_hashed_address(hashed_address); + } + + Ok(()) +} diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index 5ec5b9cdbc2..a497fa408de 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -20,8 +20,10 @@ use reth_provider::{ StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, StorageSettingsCache, }; use reth_revm::database::StateProviderDatabase; -use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot}; -use reth_trie_db::DatabaseStateRoot; +use reth_trie::{ + hashed_cursor::zero_destroyed_account_storage, HashedPostState, KeccakKeyHasher, StateRoot, +}; +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; use std::{ collections::BTreeMap, fs, @@ -256,8 +258,14 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { .map_err(|err| Error::block_failed(block_number, err))?; // Compute and check the post state root - let hashed_state = + let mut hashed_state = HashedPostState::from_bundle_state::(output.state.state()); + zero_destroyed_account_storage( + &DatabaseHashedCursorFactory::new(provider.tx_ref()), + output.state.state(), + &mut hashed_state, + ) + .map_err(|err| Error::block_failed(block_number, err))?; let sorted = hashed_state.clone_into_sorted(); let (computed_state_root, _) = reth_trie_db::with_adapter!(provider, |A| { StateRoot::, _>::overlay_root_with_updates( From 79874632215ceca7f45f59cd19d9d36d34cb58aa Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 28 Jul 2026 15:59:22 +0200 Subject: [PATCH 058/148] refactor(storage): consolidate overlay ownership (#26522) --- crates/engine/tree/src/launch.rs | 8 +- crates/engine/tree/src/tree/mod.rs | 33 ++- .../engine/tree/src/tree/payload_validator.rs | 40 +--- crates/engine/tree/src/tree/state.rs | 14 +- .../tree/src/tree/state_root_strategy/mod.rs | 44 ++-- .../tree/state_root_strategy/sparse_trie.rs | 26 +-- crates/engine/tree/src/tree/tests.rs | 22 +- crates/node/builder/src/launch/common.rs | 10 +- crates/node/builder/src/launch/engine.rs | 29 +-- crates/node/builder/src/rpc.rs | 11 +- .../src/providers/blockchain_provider.rs | 13 +- .../provider/src/providers/database/mod.rs | 35 ++-- .../src/providers/database/provider.rs | 36 ++-- .../src/providers/state/historical.rs | 143 +++++++------ .../provider/src/providers/state/overlay.rs | 18 +- crates/storage/storage-overlay/src/builder.rs | 190 +++++++----------- .../storage-overlay/src/changeset_cache.rs | 24 +-- crates/storage/storage-overlay/src/lib.rs | 2 +- crates/storage/storage-overlay/src/manager.rs | 57 +++++- crates/trie/parallel/src/proof_task.rs | 9 +- examples/custom-state-root/src/main.rs | 10 +- 21 files changed, 347 insertions(+), 427 deletions(-) diff --git a/crates/engine/tree/src/launch.rs b/crates/engine/tree/src/launch.rs index d2a6e6c9fe0..2028cd8de4a 100644 --- a/crates/engine/tree/src/launch.rs +++ b/crates/engine/tree/src/launch.rs @@ -25,7 +25,7 @@ use reth_provider::{ }; use reth_prune::PrunerWithFactory; use reth_stages_api::{MetricEventsSender, Pipeline}; -use reth_storage_overlay::{ChangesetCache, OverlayManager}; +use reth_storage_overlay::OverlayManager; use reth_tasks::Runtime; use std::sync::Arc; @@ -60,11 +60,10 @@ pub fn build_engine_orchestrator( pruner: PrunerWithFactory>, payload_builder: PayloadBuilderHandle, payload_validator: V, - state_trie_overlays: OverlayManager, + overlay_manager: OverlayManager, tree_config: TreeConfig, sync_metrics_tx: MetricEventsSender, evm_config: C, - changeset_cache: ChangesetCache, runtime: Runtime, ) -> ChainOrchestrator< EngineHandler< @@ -95,11 +94,10 @@ where persistence_handle, payload_builder, canonical_in_memory_state, - state_trie_overlays, + overlay_manager, tree_config, engine_kind, evm_config, - changeset_cache, runtime, ); diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 434226a28fb..2507d45ee72 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -36,7 +36,7 @@ use reth_provider::{ }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; -use reth_storage_overlay::{ChangesetCache, OverlayManager}; +use reth_storage_overlay::OverlayManager; use reth_tasks::{spawn_os_thread, utils::increase_thread_priority}; use reth_trie::ComputedTrieData; use revm::interpreter::debug_unreachable; @@ -99,7 +99,7 @@ pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS; /// The minimum number of blocks to retain in the changeset cache after eviction. /// -/// This ensures that recent trie changesets are kept in memory for potential reorgs, +/// This ensures that recent changesets are kept in memory for potential reorgs, /// even when the finalized block is not set (e.g., on L2s like Optimism). const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64; @@ -165,7 +165,7 @@ impl EngineApiTreeState { invalid_header_hit_eviction_threshold: u8, canonical_block: BlockNumHash, engine_kind: EngineApiKind, - state_trie_overlays: OverlayManager, + overlay_manager: OverlayManager, ) -> Self { Self { invalid_headers: InvalidHeaderCache::new( @@ -173,7 +173,7 @@ impl EngineApiTreeState { invalid_header_hit_eviction_threshold, ), buffer: BlockBuffer::new(block_buffer_limit), - tree_state: TreeState::new(canonical_block, engine_kind, state_trie_overlays), + tree_state: TreeState::new(canonical_block, engine_kind, overlay_manager), pending_sparse_trie_prune: false, forkchoice_state_tracker: ForkchoiceStateTracker::default(), } @@ -349,8 +349,6 @@ where engine_kind: EngineApiKind, /// The EVM configuration. evm_config: C, - /// Changeset cache for in-memory trie changesets - changeset_cache: ChangesetCache, /// Timing statistics for executed blocks, keyed by block hash. /// Stored here (not in `ExecutedBlock`) to avoid leaking observability concerns into the block /// type. Entries are removed when blocks are persisted or invalidated. @@ -384,7 +382,6 @@ where .field("metrics", &self.metrics) .field("engine_kind", &self.engine_kind) .field("evm_config", &self.evm_config) - .field("changeset_cache", &self.changeset_cache) .field("execution_timing_stats", &self.execution_timing_stats.len()) .field("runtime", &self.runtime) .finish() @@ -426,7 +423,6 @@ where config: TreeConfig, engine_kind: EngineApiKind, evm_config: C, - changeset_cache: ChangesetCache, runtime: reth_tasks::Runtime, ) -> Self { let (incoming_tx, incoming) = crossbeam_channel::unbounded(); @@ -448,7 +444,6 @@ where incoming_tx, engine_kind, evm_config, - changeset_cache, execution_timing_stats: B256Map::default(), building_payload: false, runtime, @@ -468,11 +463,10 @@ where persistence: PersistenceHandle, payload_builder: PayloadBuilderHandle, canonical_in_memory_state: CanonicalInMemoryState, - state_trie_overlays: OverlayManager, + overlay_manager: OverlayManager, config: TreeConfig, kind: EngineApiKind, evm_config: C, - changeset_cache: ChangesetCache, runtime: reth_tasks::Runtime, ) -> (Sender, N::Block>>, UnboundedReceiver>) { @@ -491,7 +485,7 @@ where config.invalid_header_hit_eviction_threshold(), header.num_hash(), kind, - state_trie_overlays, + overlay_manager, ); let task = Self::new( @@ -507,7 +501,6 @@ where config, kind, evm_config, - changeset_cache, runtime, ); let incoming = task.incoming_tx.clone(); @@ -1542,7 +1535,7 @@ where debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish"); self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number); - // Evict trie changesets for blocks below the eviction threshold. + // Evict cached changesets for blocks below the eviction threshold. // Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect // the finalized block if set. let min_threshold = @@ -1562,7 +1555,7 @@ where eviction_threshold, "Evicting changesets below threshold" ); - self.changeset_cache.evict(eviction_threshold); + self.state.tree_state.overlay_manager.evict_cached_changesets(eviction_threshold); self.on_new_persisted_block()?; @@ -2238,11 +2231,11 @@ where "computing block trie updates", ); let db_provider = self.provider.database_provider_ro()?; - let trie_updates = reth_storage_overlay::compute_block_trie_updates( - &self.changeset_cache, - &db_provider, - block.number(), - )?; + let trie_updates = self + .state + .tree_state + .overlay_manager + .compute_block_trie_updates(&db_provider, block.number())?; let sorted_hashed_state = Arc::new(hashed_state.into_sorted()); let sorted_trie_updates = Arc::new(trie_updates); diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 6b616481e21..3e4ac98f685 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -152,7 +152,7 @@ use reth_provider::{ StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; -use reth_storage_overlay::{ChangesetCache, OverlayBuilder, OverlayManager}; +use reth_storage_overlay::OverlayManager; use reth_trie::{ hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, LazyTrieData, @@ -284,12 +284,10 @@ where metrics: EngineApiMetrics, /// Validator for the payload. validator: V, - /// Changeset cache for in-memory trie changesets - changeset_cache: ChangesetCache, /// Task runtime for spawning parallel work. runtime: reth_tasks::Runtime, - /// Shared state trie in-memory overlay data. - state_trie_overlays: OverlayManager, + /// Shared overlay manager. + overlay_manager: OverlayManager, /// State-root strategy used to prepare per-block commitment tasks. #[debug(skip)] state_root_strategy: Arc>, @@ -337,8 +335,7 @@ where validator: V, config: TreeConfig, invalid_block_hook: Box>, - changeset_cache: ChangesetCache, - state_trie_overlays: OverlayManager, + overlay_manager: OverlayManager, runtime: reth_tasks::Runtime, ) -> Self { let precompile_cache_map = PrecompileCacheMap::default(); @@ -359,9 +356,8 @@ where invalid_block_hook, metrics: EngineApiMetrics::default(), validator, - changeset_cache, runtime, - state_trie_overlays, + overlay_manager, state_root_strategy: Arc::new(DefaultStateRootStrategy::default()), txpool_prewarm: None, } @@ -601,11 +597,7 @@ where // Create overlay factory for state-root tasks that need multiproofs. let provider_factory = self.provider.clone(); - let overlay_builder = Self::overlay_builder_for_parent( - parent_hash, - ctx.state(), - self.changeset_cache.clone(), - ); + let overlay_builder = ctx.state().tree_state.overlay_manager.overlay_builder(parent_hash); let overlay_factory = OverlayStateProviderFactory::new(provider_factory, overlay_builder); let parallel_bal_execution = ensure_ok!(self.bal_path_eligible(env.decoded_bal.as_deref())); @@ -614,7 +606,7 @@ where let mut state_root_job = ensure_ok!(self.state_root_strategy.prepare(StateRootJobContext::new( &self.runtime, - &self.state_trie_overlays, + &self.overlay_manager, &env, &parent_block, provider_builder.clone(), @@ -1449,16 +1441,6 @@ where self.invalid_block_hook.on_invalid_block(parent_header, block, output, trie_updates); } - /// Returns an overlay builder configured for a payload parent. - fn overlay_builder_for_parent( - parent_hash: B256, - state: &EngineApiTreeState, - changeset_cache: ChangesetCache, - ) -> OverlayBuilder { - OverlayBuilder::new(parent_hash, changeset_cache) - .with_overlay_manager(state.tree_state.state_trie_overlays.clone()) - } - /// Prepares the optional payload-builder state-root handle through the installed /// [`StateRootStrategy`]. fn payload_state_root_handle_for( @@ -1483,12 +1465,12 @@ where }; let overlay_factory = OverlayStateProviderFactory::new( self.provider.clone(), - Self::overlay_builder_for_parent(parent_hash, state, self.changeset_cache.clone()), + state.tree_state.overlay_manager.overlay_builder(parent_hash), ); match self.state_root_strategy.prepare_payload_builder(PayloadStateRootJobContext::new( &self.runtime, - &self.state_trie_overlays, + &self.overlay_manager, parent_hash, parent_header, timestamp, @@ -1964,7 +1946,7 @@ where debug!(target: "engine::tree::payload_validator", "Waiting for execution cache and sparse trie locks"); let execution_cache = self.payload_processor.execution_cache(); - let state_trie_overlays = self.state_trie_overlays.clone(); + let overlay_manager = self.overlay_manager.clone(); let (execution_tx, execution_rx) = std::sync::mpsc::channel(); let (sparse_trie_tx, sparse_trie_rx) = std::sync::mpsc::channel(); @@ -1972,7 +1954,7 @@ where let _ = execution_tx.send(execution_cache.wait_for_availability()); }); self.runtime.spawn_blocking_named("wait-sparse-tri", move || { - let _ = sparse_trie_tx.send(state_trie_overlays.wait_for_sparse_trie_availability()); + let _ = sparse_trie_tx.send(overlay_manager.wait_for_sparse_trie_availability()); }); let execution_cache = diff --git a/crates/engine/tree/src/tree/state.rs b/crates/engine/tree/src/tree/state.rs index 75921af45ad..a879ffd1b51 100644 --- a/crates/engine/tree/src/tree/state.rs +++ b/crates/engine/tree/src/tree/state.rs @@ -39,8 +39,8 @@ pub struct TreeState { pub(crate) current_canonical_head: BlockNumHash, /// The engine API variant of this handler pub(crate) engine_kind: EngineApiKind, - /// Flattened state trie overlays for in-memory blocks. - pub(crate) state_trie_overlays: OverlayManager, + /// Manages state trie overlays for in-memory blocks. + pub(crate) overlay_manager: OverlayManager, } impl TreeState { @@ -48,7 +48,7 @@ impl TreeState { pub fn new( current_canonical_head: BlockNumHash, engine_kind: EngineApiKind, - state_trie_overlays: OverlayManager, + overlay_manager: OverlayManager, ) -> Self { Self { blocks_by_hash: B256Map::default(), @@ -56,7 +56,7 @@ impl TreeState { current_canonical_head, parent_to_child: B256Map::default(), engine_kind, - state_trie_overlays, + overlay_manager, } } @@ -65,7 +65,7 @@ impl TreeState { let engine_kind = self.engine_kind; let removed_hashes = self.blocks_by_hash.keys().copied().collect::>(); if !removed_hashes.is_empty() { - self.state_trie_overlays.remove_blocks(removed_hashes); + self.overlay_manager.remove_blocks(removed_hashes); } self.blocks_by_hash.clear(); self.blocks_by_number.clear(); @@ -127,7 +127,7 @@ impl TreeState { self.blocks_by_number.entry(block_number).or_default().push(executed); self.parent_to_child.entry(parent_hash).or_default().insert(hash); - self.state_trie_overlays.insert_block(overlay_block); + self.overlay_manager.insert_block(overlay_block); } /// Remove single executed block by its hash. @@ -322,7 +322,7 @@ impl TreeState { } if !removed_hashes.is_empty() { - self.state_trie_overlays.remove_blocks(removed_hashes); + self.overlay_manager.remove_blocks(removed_hashes); } } diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 6e15736b43b..5de4e56d369 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -141,7 +141,7 @@ where N: NodePrimitives, { executor: &'a reth_tasks::Runtime, - state_trie_overlays: &'a OverlayManager, + overlay_manager: &'a OverlayManager, parent_hash: B256, parent_header: &'a N::BlockHeader, timestamp: u64, @@ -173,7 +173,7 @@ where #[expect(clippy::too_many_arguments)] pub(crate) const fn new( executor: &'a reth_tasks::Runtime, - state_trie_overlays: &'a OverlayManager, + overlay_manager: &'a OverlayManager, parent_hash: B256, parent_header: &'a N::BlockHeader, timestamp: u64, @@ -184,7 +184,7 @@ where ) -> Self { Self { executor, - state_trie_overlays, + overlay_manager, parent_hash, parent_header, timestamp, @@ -249,7 +249,7 @@ where Evm: ConfigureEvm, { executor: &'a reth_tasks::Runtime, - state_trie_overlays: &'a OverlayManager, + overlay_manager: &'a OverlayManager, env: &'a ExecutionEnv, parent_header: &'a SealedHeader, provider_builder: StateProviderBuilder, @@ -281,7 +281,7 @@ where #[expect(clippy::too_many_arguments)] pub(crate) const fn new( executor: &'a reth_tasks::Runtime, - state_trie_overlays: &'a OverlayManager, + overlay_manager: &'a OverlayManager, env: &'a ExecutionEnv, parent_header: &'a SealedHeader, provider_builder: StateProviderBuilder, @@ -292,7 +292,7 @@ where ) -> Self { Self { executor, - state_trie_overlays, + overlay_manager, env, parent_header, provider_builder, @@ -518,7 +518,7 @@ impl DefaultStateRootStrategy { fn spawn_state_root( &self, executor: &reth_tasks::Runtime, - state_trie_overlays: &OverlayManager, + overlay_manager: &OverlayManager, multiproof_provider_factory: F, options: StateRootTaskOptions<'_, N>, ) -> StateRootHandle @@ -553,7 +553,7 @@ impl DefaultStateRootStrategy { self.spawn_sparse_trie_task( executor, - state_trie_overlays, + overlay_manager, proof_handle, state_root_tx, hashed_state_tx, @@ -585,7 +585,7 @@ impl DefaultStateRootStrategy { fn spawn_sparse_trie_task( &self, executor: &reth_tasks::Runtime, - state_trie_overlays: &OverlayManager, + overlay_manager: &OverlayManager, proof_worker_handle: ProofWorkerHandle, state_root_tx: mpsc::Sender>, hashed_state_tx: mpsc::Sender>, @@ -599,7 +599,7 @@ impl DefaultStateRootStrategy { chunk_size, pending_sparse_trie_prune_blocks, } = options; - let state_trie_overlays = state_trie_overlays.clone(); + let overlay_manager = overlay_manager.clone(); let trie_metrics = self.metrics.clone(); let executor = executor.clone(); @@ -686,10 +686,10 @@ impl DefaultStateRootStrategy { ); let (preserved, completer) = PreservedSparseTrie::pending(result.state_root, preserved_anchor_hash); - state_trie_overlays.store_sparse_trie(preserved); + overlay_manager.store_sparse_trie(preserved); Some(completer) } else { - state_trie_overlays.clear_sparse_trie(); + overlay_manager.clear_sparse_trie(); None }; @@ -704,7 +704,7 @@ impl DefaultStateRootStrategy { "State root receiver dropped, dropping trie" ); let (trie, deferred) = task.into_cleared_trie(); - state_trie_overlays.clear_sparse_trie(); + overlay_manager.clear_sparse_trie(); executor.spawn_drop(trie); executor.spawn_drop(deferred); return; @@ -841,7 +841,7 @@ where let pending_sparse_trie_prune_blocks = ctx.take_sparse_trie_prune_blocks(); let StateRootJobContext { executor, - state_trie_overlays, + overlay_manager, env, parent_header, provider_builder, @@ -851,7 +851,7 @@ where state: _, } = ctx; - let preserved_sparse_trie = state_trie_overlays.take_sparse_trie(); + let preserved_sparse_trie = overlay_manager.take_sparse_trie(); let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie .as_ref() .filter(|trie| trie.state_root() == env.parent_state_root) @@ -864,7 +864,7 @@ where let mut handle = self.spawn_state_root( executor, - state_trie_overlays, + overlay_manager, overlay_factory.clone(), StateRootTaskOptions { parent_header: parent_header.clone(), @@ -927,7 +927,7 @@ where let pending_sparse_trie_prune_blocks = ctx.take_sparse_trie_prune_blocks(); let parent_state_root = ctx.parent_state_root(); let parent_header = SealedHeader::new(ctx.parent_header().clone(), ctx.parent_hash()); - let preserved_sparse_trie = ctx.state_trie_overlays.take_sparse_trie(); + let preserved_sparse_trie = ctx.overlay_manager.take_sparse_trie(); let overlay_factory = if let Some(anchor_hash) = preserved_sparse_trie .as_ref() .filter(|trie| trie.state_root() == parent_state_root) @@ -940,7 +940,7 @@ where Ok(Some( self.spawn_state_root( ctx.executor, - ctx.state_trie_overlays, + ctx.overlay_manager, overlay_factory, StateRootTaskOptions { parent_header, @@ -1328,7 +1328,7 @@ mod tests { test_utils::create_test_provider_factory_with_chain_spec, HashingWriter, }; - use reth_storage_overlay::{ChangesetCache, OverlayBuilder, OverlayManager}; + use reth_storage_overlay::OverlayManager; use reth_testing_utils::generators; use reth_trie::test_utils::state_root; use revm::state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot, TransactionId}; @@ -1482,13 +1482,13 @@ mod tests { let provider_factory = BlockchainProvider::new(factory).unwrap(); let env: ExecutionEnv = ExecutionEnv::test_default(); let runtime = reth_tasks::Runtime::test(); - let state_trie_overlays = OverlayManager::::default(); + let overlay_manager = OverlayManager::::default(); let mut state_root_handle = DefaultStateRootStrategy::default().spawn_state_root( &runtime, - &state_trie_overlays, + &overlay_manager, OverlayStateProviderFactory::new( provider_factory, - OverlayBuilder::::new(genesis_hash, ChangesetCache::new()), + overlay_manager.overlay_builder(genesis_hash), ), StateRootTaskOptions { parent_header: SealedHeader::new(Default::default(), genesis_hash), diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index e754a07577a..982df30d207 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -1121,7 +1121,7 @@ mod tests { providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory, ChainSpecProvider, }; - use reth_storage_overlay::{ChangesetCache, OverlayBuilder}; + use reth_storage_overlay::OverlayManager; use reth_trie_parallel::proof_task::ProofTaskCtx; use reth_trie_sparse::ArenaParallelSparseTrie; @@ -1225,10 +1225,8 @@ mod tests { let anchor_hash = provider_factory.chain_spec().genesis_hash(); let overlay_factory = OverlayStateProviderFactory::new( provider_factory, - OverlayBuilder::::new( - anchor_hash, - ChangesetCache::new(), - ), + OverlayManager::::default() + .overlay_builder(anchor_hash), ); let proof_worker_handle = ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); @@ -1272,10 +1270,8 @@ mod tests { let anchor_hash = provider_factory.chain_spec().genesis_hash(); let overlay_factory = OverlayStateProviderFactory::new( provider_factory, - OverlayBuilder::::new( - anchor_hash, - ChangesetCache::new(), - ), + OverlayManager::::default() + .overlay_builder(anchor_hash), ); let proof_worker_handle = ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); @@ -1352,10 +1348,8 @@ mod tests { let anchor_hash = provider_factory.chain_spec().genesis_hash(); let overlay_factory = OverlayStateProviderFactory::new( provider_factory, - OverlayBuilder::::new( - anchor_hash, - ChangesetCache::new(), - ), + OverlayManager::::default() + .overlay_builder(anchor_hash), ); let proof_worker_handle = ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); @@ -1398,10 +1392,8 @@ mod tests { let anchor_hash = provider_factory.chain_spec().genesis_hash(); let overlay_factory = OverlayStateProviderFactory::new( provider_factory, - OverlayBuilder::::new( - anchor_hash, - ChangesetCache::new(), - ), + OverlayManager::::default() + .overlay_builder(anchor_hash), ); let proof_worker_handle = ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index a9323b8e948..a6cd24e5cbd 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -7,7 +7,7 @@ use crate::{ PersistTarget, TreeConfig, }, }; -use reth_storage_overlay::{ChangesetCache, OverlayManager}; +use reth_storage_overlay::OverlayManager; use alloy_eips::eip1898::BlockWithParent; use alloy_primitives::{ @@ -206,7 +206,7 @@ impl TestHarness { let (from_tree_tx, from_tree_rx) = unbounded_channel(); let runtime = reth_tasks::Runtime::test(); - let state_trie_overlays = OverlayManager::new(runtime.state_trie_overlay_worker_pool()); + let overlay_manager = OverlayManager::new(runtime.state_trie_overlay_worker_pool()); let header = chain_spec.genesis_header().clone(); let header = SealedHeader::seal_slow(header); @@ -216,7 +216,7 @@ impl TestHarness { tree_config.invalid_header_hit_eviction_threshold(), header.num_hash(), EngineApiKind::Ethereum, - state_trie_overlays.clone(), + overlay_manager.clone(), ); let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None); @@ -224,7 +224,6 @@ impl TestHarness { let payload_builder = PayloadBuilderHandle::new(to_payload_service); let evm_config = MockEvmConfig::default(); - let changeset_cache = ChangesetCache::new(); let engine_validator = BasicEngineValidator::new( provider.clone(), consensus.clone(), @@ -232,8 +231,7 @@ impl TestHarness { payload_validator, tree_config.clone(), Box::new(NoopInvalidBlockHook::default()), - changeset_cache.clone(), - state_trie_overlays, + overlay_manager, runtime.clone(), ); @@ -250,7 +248,6 @@ impl TestHarness { tree_config, EngineApiKind::Ethereum, evm_config, - changeset_cache, runtime, ); @@ -287,9 +284,9 @@ impl TestHarness { parent_hash = hash; } - let state_trie_overlays = OverlayManager::default(); + let overlay_manager = self.tree.state.tree_state.overlay_manager.clone(); for block in &blocks { - state_trie_overlays.insert_block(block.clone()); + overlay_manager.insert_block(block.clone()); } self.tree.state.tree_state = TreeState { @@ -298,7 +295,7 @@ impl TestHarness { current_canonical_head: blocks.last().unwrap().recovered_block().num_hash(), parent_to_child, engine_kind: EngineApiKind::Ethereum, - state_trie_overlays, + overlay_manager, }; let last_executed_block = blocks.last().unwrap().clone(); @@ -431,7 +428,7 @@ impl ValidatorTestHarness { let provider = harness.provider.clone(); let payload_validator = MockEngineValidator; let evm_config = MockEvmConfig::default(); - let changeset_cache = ChangesetCache::new(); + let overlay_manager = harness.tree.state.tree_state.overlay_manager.clone(); let validator = BasicEngineValidator::new( provider, @@ -440,8 +437,7 @@ impl ValidatorTestHarness { payload_validator, TreeConfig::default(), Box::new(NoopInvalidBlockHook::default()), - changeset_cache, - OverlayManager::default(), + overlay_manager, reth_tasks::Runtime::test(), ); diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 857bdd4eb6e..cdf3f978543 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -81,7 +81,7 @@ use reth_stages::{ StageId, StageSet, }; use reth_static_file::{blocks_per_file_for_prune_distance, StaticFileProducer, StaticFileSegment}; -use reth_storage_overlay::ChangesetCache; +use reth_storage_overlay::OverlayManager; use reth_tasks::TaskExecutor; use reth_tracing::{ throttle, @@ -475,7 +475,7 @@ where /// check.** pub async fn create_provider_factory( &self, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, rocksdb_provider: Option, disabled_stages: &[StageId], ) -> eyre::Result> @@ -537,7 +537,7 @@ where )? .with_prune_modes(prune_config.segments) .with_minimum_pruning_distance(prune_config.minimum_pruning_distance) - .with_changeset_cache(changeset_cache) + .with_overlay_manager(overlay_manager) .with_bal_store(bal_store); // Check consistency between the database and static files, returning @@ -610,7 +610,7 @@ where /// Creates a new [`ProviderFactory`] and attaches it to the launch context. pub async fn with_provider_factory( self, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, rocksdb_provider: Option, disabled_stages: &[StageId], ) -> eyre::Result, ProviderFactory>>> @@ -619,7 +619,7 @@ where Evm: ConfigureEvm + 'static, { let factory = self - .create_provider_factory::(changeset_cache, rocksdb_provider, disabled_stages) + .create_provider_factory::(overlay_manager, rocksdb_provider, disabled_stages) .await?; let ctx = LaunchContextWith { inner: self.inner, diff --git a/crates/node/builder/src/launch/engine.rs b/crates/node/builder/src/launch/engine.rs index 57e6737f6a1..c47b5ed6bb3 100644 --- a/crates/node/builder/src/launch/engine.rs +++ b/crates/node/builder/src/launch/engine.rs @@ -37,7 +37,7 @@ use reth_provider::{ providers::{BlockchainProvider, NodeTypesForProvider}, BlockNumReader, StorageSettingsCache, }; -use reth_storage_overlay::{ChangesetCache, OverlayManager}; +use reth_storage_overlay::OverlayManager; use reth_tasks::TaskExecutor; use reth_tokio_util::EventSender; use reth_tracing::tracing::{debug, error, info}; @@ -92,8 +92,10 @@ impl EngineNodeLauncher { } = target; let NodeHooks { on_component_initialized, on_node_started, .. } = hooks; - // Create changeset cache that will be shared across the engine - let changeset_cache = ChangesetCache::new(); + // Create the overlay manager that will be shared across the provider and engine. + let overlay_manager = OverlayManager::::new( + ctx.task_executor.state_trie_overlay_worker_pool(), + ); let disabled_stages = N::disabled_stages(); // setup the launch context @@ -107,9 +109,9 @@ impl EngineNodeLauncher { .attach(database.clone()) // ensure certain settings take effect .with_adjusted_configs() - // Create the provider factory with changeset cache + // Create the provider factory with the shared overlay manager .with_provider_factory::<_, >::Evm>( - changeset_cache.clone(), + overlay_manager.clone(), rocksdb_provider, disabled_stages, ) @@ -203,22 +205,15 @@ impl EngineNodeLauncher { engine_events: event_sender.clone(), }; let validator_builder = add_ons.engine_validator_builder(); - let state_trie_overlays = - OverlayManager::new(ctx.task_executor().state_trie_overlay_worker_pool()); // Build the engine validator with all required components let engine_validator = validator_builder .clone() - .build_tree_validator( - &add_ons_ctx, - engine_tree_config.clone(), - changeset_cache.clone(), - state_trie_overlays.clone(), - ) + .build_tree_validator(&add_ons_ctx, engine_tree_config.clone(), overlay_manager.clone()) .await?; // Create the consensus engine stream with optional reorg - let reorg_state_trie_overlays = state_trie_overlays.clone(); + let reorg_overlay_manager = overlay_manager.clone(); let consensus_engine_stream = UnboundedReceiverStream::from(consensus_engine_rx) .maybe_skip_fcu(node_config.debug.skip_fcu) .maybe_skip_new_payload(node_config.debug.skip_new_payload) @@ -230,8 +225,7 @@ impl EngineNodeLauncher { .build_tree_validator( &add_ons_ctx, engine_tree_config.clone(), - changeset_cache.clone(), - reorg_state_trie_overlays.clone(), + reorg_overlay_manager.clone(), ) .await }, @@ -262,11 +256,10 @@ impl EngineNodeLauncher { pruner, ctx.components().payload_builder_handle().clone(), engine_validator, - state_trie_overlays, + overlay_manager, engine_tree_config, ctx.sync_metrics_tx(), ctx.components().evm_config().clone(), - changeset_cache, ctx.task_executor().clone(), ); diff --git a/crates/node/builder/src/rpc.rs b/crates/node/builder/src/rpc.rs index bcf0942c7a1..47af03d871d 100644 --- a/crates/node/builder/src/rpc.rs +++ b/crates/node/builder/src/rpc.rs @@ -10,7 +10,7 @@ pub use reth_rpc_builder::{ middleware::{RethAuthHttpMiddleware, RethRpcMiddleware}, Identity, Stack, }; -use reth_storage_overlay::{ChangesetCache, OverlayManager}; +use reth_storage_overlay::OverlayManager; use crate::{ invalid_block_hook::InvalidBlockHookExt, txpool_prewarm, ConfigureEngineEvm, @@ -1412,8 +1412,7 @@ pub trait EngineValidatorBuilder: Send + Sync + Clone self, ctx: &AddOnsContext<'_, Node>, tree_config: TreeConfig, - changeset_cache: ChangesetCache, - state_trie_overlays: OverlayManager>, + overlay_manager: OverlayManager>, ) -> impl Future> + Send; } @@ -1461,8 +1460,7 @@ where self, ctx: &AddOnsContext<'_, Node>, tree_config: TreeConfig, - changeset_cache: ChangesetCache, - state_trie_overlays: OverlayManager>, + overlay_manager: OverlayManager>, ) -> eyre::Result { let validator = self.payload_validator_builder.build(ctx).await?; let data_dir = ctx.config.datadir.clone().resolve_datadir(ctx.config.chain.chain()); @@ -1476,8 +1474,7 @@ where validator, tree_config, invalid_block_hook, - changeset_cache, - state_trie_overlays, + overlay_manager, ctx.node.task_executor().clone(), ); diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 23b073419f4..1b1b93bda8b 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -35,7 +35,6 @@ use reth_storage_api::{ StorageRangeResult, }; use reth_storage_errors::provider::ProviderResult; -use reth_storage_overlay::OverlayBuilder; use reth_trie::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, metrics::TrieRootMetrics, @@ -187,12 +186,10 @@ impl BlockchainProvider { // via changesets, then the merged in-memory delta applies on top. let overlay_factory = OverlayStateProviderFactory::new( self.database.clone(), - OverlayBuilder::::new( - matched.anchor().hash, - self.database.changeset_cache(), - ) - .with_hashed_state_overlay(Some(merged.state)) - .with_trie_updates_overlay(Some(merged.nodes)), + self.database + .overlay_manager() + .overlay_builder(matched.anchor().hash) + .with_immediate_state_trie_overlay(merged.state, merged.nodes), ); reth_storage_api::DatabaseProviderROFactory::database_provider_ro(&overlay_factory) .map(Some) @@ -220,7 +217,7 @@ impl BlockchainProvider { let Some(block_hash) = block_hash else { return Ok(None) }; let overlay_factory = OverlayStateProviderFactory::new( self.database.clone(), - OverlayBuilder::::new(block_hash, self.database.changeset_cache()), + self.database.overlay_manager().overlay_builder(block_hash), ); reth_storage_api::DatabaseProviderROFactory::database_provider_ro(&overlay_factory) .map(Some) diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index ccd6e59df55..8686d51728e 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -33,7 +33,7 @@ use reth_storage_api::{ NodePrimitivesProvider, StorageSettings, StorageSettingsCache, TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::ProviderResult; -use reth_storage_overlay::ChangesetCache; +use reth_storage_overlay::OverlayManager; use reth_trie::HashedPostState; use revm::database::BundleState; use std::{ @@ -89,8 +89,8 @@ pub struct ProviderFactory { storage_settings: Arc>, /// `RocksDB` provider rocksdb_provider: RocksDBProvider, - /// Changeset cache for trie unwinding - changeset_cache: ChangesetCache, + /// Manager for state trie overlays and cached changesets. + overlay_manager: OverlayManager, /// Store for block access lists. bal_store: BalStoreHandle, /// Task runtime for spawning parallel I/O work. @@ -134,6 +134,7 @@ impl ProviderFactory { // Both factory and all providers it creates should share these cached settings. let legacy_settings = StorageSettings::v1(); let database_provider_metrics = Arc::new(DatabaseProviderMetrics::default()); + let overlay_manager = OverlayManager::default(); let storage_settings = DatabaseProvider::<_, N>::new( db.tx()?, chain_spec.clone(), @@ -142,7 +143,7 @@ impl ProviderFactory { Default::default(), Arc::new(RwLock::new(legacy_settings)), rocksdb_provider.clone(), - ChangesetCache::new(), + overlay_manager.clone(), runtime.clone(), db.path(), database_provider_metrics.clone(), @@ -158,7 +159,7 @@ impl ProviderFactory { storage: Default::default(), storage_settings: Arc::new(RwLock::new(storage_settings)), rocksdb_provider, - changeset_cache: ChangesetCache::new(), + overlay_manager, bal_store: BalStoreHandle::new(InMemoryBalStore::default()), runtime, minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE, @@ -198,15 +199,15 @@ impl ProviderFactory { self } - /// Sets the changeset cache for an existing [`ProviderFactory`]. - pub fn with_changeset_cache(mut self, changeset_cache: ChangesetCache) -> Self { - self.changeset_cache = changeset_cache; + /// Sets the overlay manager for an existing [`ProviderFactory`]. + pub fn with_overlay_manager(mut self, overlay_manager: OverlayManager) -> Self { + self.overlay_manager = overlay_manager; self } - /// Returns the shared changeset cache. - pub(crate) fn changeset_cache(&self) -> ChangesetCache { - self.changeset_cache.clone() + /// Returns the shared overlay manager. + pub(crate) const fn overlay_manager(&self) -> &OverlayManager { + &self.overlay_manager } /// Sets the minimum pruning distance for an existing [`ProviderFactory`]. @@ -398,7 +399,7 @@ impl ProviderFactory { self.storage.clone(), self.storage_settings.clone(), self.rocksdb_provider.clone(), - self.changeset_cache.clone(), + self.overlay_manager.clone(), self.runtime.clone(), self.db.path(), self.database_provider_metrics.clone(), @@ -421,7 +422,7 @@ impl ProviderFactory { self.storage.clone(), self.storage_settings.clone(), self.rocksdb_provider.clone(), - self.changeset_cache.clone(), + self.overlay_manager.clone(), self.runtime.clone(), self.db.path(), self.database_provider_metrics.clone(), @@ -449,7 +450,7 @@ impl ProviderFactory { self.storage.clone(), self.storage_settings.clone(), self.rocksdb_provider.clone(), - self.changeset_cache.clone(), + self.overlay_manager.clone(), self.runtime.clone(), self.db.path(), self.database_provider_metrics.clone(), @@ -983,7 +984,7 @@ where storage, storage_settings, rocksdb_provider, - changeset_cache, + overlay_manager, bal_store, runtime, minimum_pruning_distance, @@ -998,7 +999,7 @@ where .field("storage", &storage) .field("storage_settings", &*storage_settings.read()) .field("rocksdb_provider", &rocksdb_provider) - .field("changeset_cache", &changeset_cache) + .field("overlay_manager", &overlay_manager) .field("bal_store", &bal_store) .field("runtime", &runtime) .field("minimum_pruning_distance", &minimum_pruning_distance) @@ -1020,7 +1021,7 @@ impl Clone for ProviderFactory { storage: self.storage.clone(), storage_settings: self.storage_settings.clone(), rocksdb_provider: self.rocksdb_provider.clone(), - changeset_cache: self.changeset_cache.clone(), + overlay_manager: self.overlay_manager.clone(), bal_store: self.bal_store.clone(), runtime: self.runtime.clone(), minimum_pruning_distance: self.minimum_pruning_distance, diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 52bc6210ade..033f6190374 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -65,7 +65,7 @@ use reth_storage_api::{ StoragePath, StorageSettingsCache, TryIntoHistoricalStateProvider, WriteStateInput, }; use reth_storage_errors::provider::{ProviderResult, StaticFileWriterError}; -use reth_storage_overlay::ChangesetCache; +use reth_storage_overlay::OverlayManager; use reth_trie::{ updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, ComputedTrieData, HashedPostStateSorted, @@ -200,8 +200,8 @@ pub struct DatabaseProvider { storage_settings: Arc>, /// `RocksDB` provider rocksdb_provider: RocksDBProvider, - /// Changeset cache for trie unwinding - changeset_cache: ChangesetCache, + /// Manager for state trie overlays and cached changesets. + overlay_manager: OverlayManager, /// Task runtime for spawning parallel I/O work. runtime: reth_tasks::Runtime, /// Path to the database directory. @@ -228,7 +228,7 @@ impl Debug for DatabaseProvider { .field("storage", &self.storage) .field("storage_settings", &self.storage_settings) .field("rocksdb_provider", &self.rocksdb_provider) - .field("changeset_cache", &self.changeset_cache) + .field("overlay_manager", &self.overlay_manager) .field("runtime", &self.runtime) .field("pending_rocksdb_batches", &"") .field("commit_order", &self.commit_order) @@ -327,7 +327,7 @@ impl DatabaseProvider { self.get_prune_checkpoint(PruneSegment::StorageHistory)?; let mut state_provider = - HistoricalStateProviderRef::new(self, block_number, self.changeset_cache.clone()); + HistoricalStateProviderRef::new(self, block_number, self.overlay_manager.clone()); // If we pruned account or storage history, we can't return state on every historical block. // Instead, we should cap it at the latest prune checkpoint for corresponding prune segment. if let Some(prune_checkpoint_block_number) = @@ -414,7 +414,7 @@ impl DatabaseProvider { storage: Arc, storage_settings: Arc>, rocksdb_provider: RocksDBProvider, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, runtime: reth_tasks::Runtime, db_path: PathBuf, commit_order: CommitOrder, @@ -428,7 +428,7 @@ impl DatabaseProvider { storage, storage_settings, rocksdb_provider, - changeset_cache, + overlay_manager, runtime, db_path, pending_rocksdb_batches: Default::default(), @@ -449,7 +449,7 @@ impl DatabaseProvider { storage: Arc, storage_settings: Arc>, rocksdb_provider: RocksDBProvider, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, runtime: reth_tasks::Runtime, db_path: PathBuf, metrics: Arc, @@ -462,7 +462,7 @@ impl DatabaseProvider { storage, storage_settings, rocksdb_provider, - changeset_cache, + overlay_manager, runtime, db_path, CommitOrder::Normal, @@ -480,7 +480,7 @@ impl DatabaseProvider { storage: Arc, storage_settings: Arc>, rocksdb_provider: RocksDBProvider, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, runtime: reth_tasks::Runtime, db_path: PathBuf, metrics: Arc, @@ -493,7 +493,7 @@ impl DatabaseProvider { storage, storage_settings, rocksdb_provider, - changeset_cache, + overlay_manager, runtime, db_path, CommitOrder::Unwind, @@ -877,7 +877,9 @@ impl DatabaseProvider TryIntoHistoricalStateProvider for Databa self.get_prune_checkpoint(PruneSegment::AccountHistory)?; let storage_history_prune_checkpoint = self.get_prune_checkpoint(PruneSegment::StorageHistory)?; - let changeset_cache = self.changeset_cache.clone(); + let overlay_manager = self.overlay_manager.clone(); - let mut state_provider = HistoricalStateProvider::new(self, block_number, changeset_cache); + let mut state_provider = HistoricalStateProvider::new(self, block_number, overlay_manager); // If we pruned account or storage history, we can't return state on every historical block. // Instead, we should cap it at the latest prune checkpoint for corresponding prune segment. @@ -1047,7 +1049,7 @@ impl DatabaseProvider { storage: Arc, storage_settings: Arc>, rocksdb_provider: RocksDBProvider, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, runtime: reth_tasks::Runtime, db_path: PathBuf, metrics: Arc, @@ -1060,7 +1062,7 @@ impl DatabaseProvider { storage, storage_settings, rocksdb_provider, - changeset_cache, + overlay_manager, runtime, db_path, pending_rocksdb_batches: Default::default(), @@ -5009,7 +5011,7 @@ mod tests { assert_eq!(account_cs[0].address, address); let historical_value = - HistoricalStateProviderRef::new(&*provider_rw, 0, ChangesetCache::new()) + HistoricalStateProviderRef::new(&*provider_rw, 0, OverlayManager::default()) .storage(address, slot_key) .unwrap(); assert_eq!(historical_value, None); diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index a2211941b77..8d36d979479 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -18,7 +18,7 @@ use reth_storage_api::{ StorageSettingsCache, }; use reth_storage_errors::provider::ProviderResult; -use reth_storage_overlay::{ChangesetCache, Overlay, OverlayBuilder, OverlaySource}; +use reth_storage_overlay::{Overlay, OverlayManager}; use reth_trie::{ hashed_cursor::HashedPostStateCursorFactory, proof::{Proof, StorageProof}, @@ -31,7 +31,7 @@ use reth_trie::{ }; use reth_trie_db::{DatabaseProof, DatabaseStateRoot, DatabaseStorageProof, DatabaseStorageRoot}; -use std::{fmt::Debug, marker::PhantomData, sync::Arc}; +use std::{fmt::Debug, sync::Arc}; type DbStateRoot<'a, TX, A> = StateRoot< reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>, @@ -128,14 +128,12 @@ pub struct HistoricalStateProviderRef< { /// Database provider provider: &'b Provider, - /// Changeset cache handle for retrieving trie changesets. - changeset_cache: ChangesetCache, + /// Manager for state trie overlays and cached changesets. + overlay_manager: OverlayManager, /// Block number is main index for the history state of accounts and storages. block_number: BlockNumber, /// Lowest blocks at which different parts of the state are available. lowest_available_blocks: LowestAvailableBlocks, - /// Marker for the provider's node primitives. - _primitives: PhantomData, } impl<'b, Provider, N> HistoricalStateProviderRef<'b, Provider, N> @@ -151,14 +149,13 @@ where pub fn new( provider: &'b Provider, block_number: BlockNumber, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, ) -> Self { Self { provider, - changeset_cache, + overlay_manager, block_number, lowest_available_blocks: Default::default(), - _primitives: PhantomData, } } @@ -168,15 +165,9 @@ where provider: &'b Provider, block_number: BlockNumber, lowest_available_blocks: LowestAvailableBlocks, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, ) -> Self { - Self { - provider, - changeset_cache, - block_number, - lowest_available_blocks, - _primitives: PhantomData, - } + Self { provider, overlay_manager, block_number, lowest_available_blocks } } /// Lookup an account in the `AccountsHistory` table using `EitherReader`. @@ -307,8 +298,10 @@ where .ok_or_else(|| ProviderError::HeaderNotFound(target_block.into()))?; let TrieInputSorted { nodes, state, prefix_sets } = input; - let overlay_builder = OverlayBuilder::::new(anchor_hash, self.changeset_cache.clone()) - .with_overlay_source(Some(OverlaySource::Immediate { trie: nodes, state })); + let overlay_builder = self + .overlay_manager + .overlay_builder(anchor_hash) + .with_immediate_state_trie_overlay(state, nodes); let Overlay { trie_updates, hashed_post_state } = overlay_builder.build_overlay(self.provider)?; @@ -684,29 +677,34 @@ where /// State provider for a given block number. /// For more detailed description, see [`HistoricalStateProviderRef`]. #[derive(Debug)] -pub struct HistoricalStateProvider { +pub struct HistoricalStateProvider { /// Database provider. provider: Provider, - /// Changeset cache handle for retrieving trie changesets. - changeset_cache: ChangesetCache, + /// Manager for state trie overlays and cached changesets. + overlay_manager: OverlayManager, /// State at the block number is the main indexer of the state. block_number: BlockNumber, /// Lowest blocks at which different parts of the state are available. lowest_available_blocks: LowestAvailableBlocks, } -impl - HistoricalStateProvider +impl< + Provider: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + NodePrimitivesProvider, + > HistoricalStateProvider { /// Create new `StateProvider` for historical block number pub fn new( provider: Provider, block_number: BlockNumber, - changeset_cache: ChangesetCache, + overlay_manager: OverlayManager, ) -> Self { Self { provider, - changeset_cache, + overlay_manager, block_number, lowest_available_blocks: Default::default(), } @@ -746,7 +744,7 @@ impl< &self.provider, self.block_number, self.lowest_available_blocks, - self.changeset_cache.clone(), + self.overlay_manager.clone(), ) } } @@ -889,7 +887,7 @@ mod tests { StorageChangeSetReader, StorageSettingsCache, }; use reth_storage_errors::provider::ProviderError; - use reth_storage_overlay::ChangesetCache; + use reth_storage_overlay::OverlayManager; const ADDRESS: Address = address!("0x0000000000000000000000000000000000000001"); const HIGHER_ADDRESS: Address = address!("0x0000000000000000000000000000000000000005"); @@ -980,49 +978,50 @@ mod tests { // run assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) + .basic_account(&ADDRESS), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 2, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 2, OverlayManager::default()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at3 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 3, OverlayManager::default()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at3 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 4, OverlayManager::default()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at7 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 7, OverlayManager::default()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at7 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 9, OverlayManager::default()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at10 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 10, OverlayManager::default()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at10 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 11, OverlayManager::default()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_at15 )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).basic_account(&ADDRESS), + HistoricalStateProviderRef::new(&db, 16, OverlayManager::default()).basic_account(&ADDRESS), Ok(Some(acc)) if acc == acc_plain )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) .basic_account(&HIGHER_ADDRESS), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).basic_account(&HIGHER_ADDRESS), + HistoricalStateProviderRef::new(&db, 1000, OverlayManager::default()).basic_account(&HIGHER_ADDRESS), Ok(Some(acc)) if acc == higher_acc_plain )); } @@ -1081,46 +1080,46 @@ mod tests { // run assert!(matches!( - HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 0, OverlayManager::default()) .storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 3, OverlayManager::default()) .storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 4, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 7, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 9, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 10, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 11, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at15.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 16, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_plain.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) .storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1000, OverlayManager::default()).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == higher_entry_plain.value )); } @@ -1139,7 +1138,7 @@ mod tests { account_history_block_number: Some(3), storage_history_block_number: Some(3), }, - ChangesetCache::new(), + OverlayManager::default(), ); assert!(matches!( provider.account_history_lookup(ADDRESS), @@ -1159,7 +1158,7 @@ mod tests { account_history_block_number: Some(2), storage_history_block_number: Some(2), }, - ChangesetCache::new(), + OverlayManager::default(), ); assert!(matches!( provider.account_history_lookup(ADDRESS), @@ -1179,7 +1178,7 @@ mod tests { account_history_block_number: Some(1), storage_history_block_number: Some(1), }, - ChangesetCache::new(), + OverlayManager::default(), ); assert!(matches!( provider.account_history_lookup(ADDRESS), @@ -1260,46 +1259,46 @@ mod tests { let db = factory.provider().unwrap(); assert!(matches!( - HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 0, OverlayManager::default()) .storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 3, OverlayManager::default()) .storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 4, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 7, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at7.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 9, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 10, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at10.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 11, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_at15.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 16, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == entry_plain.value )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) .storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1000, OverlayManager::default()).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(expected_value)) if expected_value == higher_entry_plain.value )); } @@ -1403,46 +1402,46 @@ mod tests { let db = factory.provider().unwrap(); assert!(matches!( - HistoricalStateProviderRef::new(&db, 0, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 0, OverlayManager::default()) .storage(ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 3, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 3, OverlayManager::default()) .storage(ADDRESS, STORAGE), Ok(Some(U256::ZERO)) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 4, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 4, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(7) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 7, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 7, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(7) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 9, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 9, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(10) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 10, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 10, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(10) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 11, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 11, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(15) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 16, ChangesetCache::new()).storage(ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 16, OverlayManager::default()).storage(ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(100) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1, ChangesetCache::new()) + HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()) .storage(HIGHER_ADDRESS, STORAGE), Ok(None) )); assert!(matches!( - HistoricalStateProviderRef::new(&db, 1000, ChangesetCache::new()).storage(HIGHER_ADDRESS, STORAGE), + HistoricalStateProviderRef::new(&db, 1000, OverlayManager::default()).storage(HIGHER_ADDRESS, STORAGE), Ok(Some(v)) if v == U256::from(1000) )); } diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index c8a4cd487fe..1d4aef1902f 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -69,16 +69,6 @@ impl OverlayStateProviderFactory { } } - /// Set the hashed state overlay. - pub fn with_hashed_state_overlay( - mut self, - hashed_state_overlay: Option>, - ) -> Self { - self.overlay_builder = self.overlay_builder.with_hashed_state_overlay(hashed_state_overlay); - self.overlay_cache = Default::default(); - self - } - /// Skips managed overlay construction when this factory is used by a task that reused a sparse /// trie covering both durable frontiers through the parent. pub fn with_skip_overlay_for_reused_sparse_trie(mut self, anchor_hash: B256) -> Self { @@ -287,7 +277,7 @@ mod tests { use reth_primitives_traits::Account; use reth_stages_types::{FinishCheckpoint, StageCheckpoint, StageId}; use reth_storage_api::StageCheckpointWriter; - use reth_storage_overlay::{ChangesetCache, OverlayManager}; + use reth_storage_overlay::OverlayManager; use reth_trie::{BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles}; fn with_unique_trie_data( @@ -367,11 +357,7 @@ mod tests { } let overlay_factory = OverlayStateProviderFactory::new( factory.clone(), - OverlayBuilder::::new( - blocks[3].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_overlay_manager(manager), + manager.overlay_builder(blocks[3].recovered_block().hash()), ); let provider = factory.provider().unwrap(); diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 80e4fb39622..e894a25b0a2 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -1,4 +1,4 @@ -use crate::{ChangesetCache, OverlayManager}; +use crate::OverlayManager; use alloy_eips::BlockNumHash; use alloy_primitives::{BlockHash, BlockNumber, B256}; use metrics::{Counter, Histogram}; @@ -40,7 +40,7 @@ impl Overlay { /// Source of data to apply on top of the durable database state. #[derive(Debug, Clone)] -pub enum OverlaySource { +pub enum OverlaySource { /// Immediate overlay with already-computed data. Immediate { /// Trie updates overlay. @@ -52,24 +52,21 @@ pub enum OverlaySource { state: Arc, }, /// Manager-backed overlay for in-memory state. - Managed { - /// Manager used to resolve in-memory parent state if the parent is not persisted. - manager: OverlayManager, - }, + Managed, } /// Builder for calculating trie and hashed-state overlays. /// -/// This stores the overlay configuration and the logic for resolving overlays and collecting -/// reverts. It is independent from any provider factory or overlay cache. +/// This stores the overlay manager, overlay configuration, and the logic for resolving overlays +/// and collecting reverts. #[derive(Debug, Clone)] pub struct OverlayBuilder { /// Parent hash requested by the caller. parent_hash: B256, /// Optional overlay source. - overlay_source: Option>, - /// Changeset cache handle for retrieving trie changesets. - changeset_cache: ChangesetCache, + overlay_source: Option, + /// Manager used for cached changesets and in-memory parent state. + overlay_manager: OverlayManager, /// Anchor hash of the reused sparse trie, if this task reused one. reused_sparse_trie_anchor_hash: Option, /// Metrics for overlay construction. @@ -77,12 +74,12 @@ pub struct OverlayBuilder { } impl OverlayBuilder { - /// Create a new overlay builder. - pub fn new(parent_hash: B256, changeset_cache: ChangesetCache) -> Self { + /// Create a new manager-backed overlay builder. + pub(crate) fn new(parent_hash: B256, overlay_manager: OverlayManager) -> Self { Self { parent_hash, - overlay_source: None, - changeset_cache, + overlay_source: Some(OverlaySource::Managed), + overlay_manager, reused_sparse_trie_anchor_hash: None, metrics: OverlayBuilderMetrics::default(), } @@ -91,7 +88,7 @@ impl OverlayBuilder { /// Set the overlay source. /// /// This overlay will be applied on top of any reverts. - pub fn with_overlay_source(mut self, source: Option>) -> Self { + pub fn with_overlay_source(mut self, source: Option) -> Self { self.overlay_source = source; self } @@ -103,51 +100,13 @@ impl OverlayBuilder { self } - /// Set the overlay manager used to resolve in-memory parent state. - pub fn with_overlay_manager(mut self, overlay_manager: OverlayManager) -> Self { - self.overlay_source = Some(OverlaySource::Managed { manager: overlay_manager }); - self - } - - /// Set the hashed state overlay. - pub fn with_hashed_state_overlay( + /// Sets an immediate hashed-state and trie-updates overlay. + pub fn with_immediate_state_trie_overlay( mut self, - hashed_state_overlay: Option>, - ) -> Self { - if let Some(new_state) = hashed_state_overlay { - match &mut self.overlay_source { - Some(OverlaySource::Immediate { state, .. }) => *state = new_state, - Some(OverlaySource::Managed { .. }) | None => { - self.overlay_source = Some(OverlaySource::Immediate { - trie: Arc::new(TrieUpdatesSorted::default()), - state: new_state, - }); - } - } - } - self - } - - /// Set the trie updates overlay. - /// - /// Only applies to an immediate overlay: a managed overlay's trie updates are resolved from - /// its manager instead. - pub fn with_trie_updates_overlay( - mut self, - trie_updates_overlay: Option>, + state: Arc, + trie: Arc, ) -> Self { - if let Some(trie) = trie_updates_overlay { - match &mut self.overlay_source { - Some(OverlaySource::Immediate { trie: existing, .. }) => *existing = trie, - Some(OverlaySource::Managed { .. }) => {} - None => { - self.overlay_source = Some(OverlaySource::Immediate { - trie, - state: Arc::new(HashedPostStateSorted::default()), - }); - } - } - } + self.overlay_source = Some(OverlaySource::Immediate { trie, state }); self } @@ -196,14 +155,14 @@ impl OverlayBuilder { let trie_updates_total_len; let hashed_state_updates_total_len; let anchor_hash = match &self.overlay_source { - Some(OverlaySource::Managed { manager }) => { + Some(OverlaySource::Managed) => { let parent_is_persisted = provider .convert_hash_or_number(self.parent_hash.into())? .is_some_and(|parent_number| parent_number <= state_trie_tip_block.number); if parent_is_persisted { self.parent_hash } else { - manager + self.overlay_manager .anchor_for_parent(self.parent_hash, state_trie_tip_block.hash) .ok_or(ProviderError::BlockHashNotFound(self.parent_hash))? } @@ -226,8 +185,9 @@ impl OverlayBuilder { let _guard = debug_span!(target: "storage::overlay", "retrieving_trie_reverts").entered(); let start = Instant::now(); - let accumulated_reverts = - self.changeset_cache.get_or_compute_range(provider, revert_blocks.clone())?; + let accumulated_reverts = self + .overlay_manager + .get_or_compute_cached_changesets_range(provider, revert_blocks.clone())?; retrieve_trie_reverts_duration = start.elapsed(); accumulated_reverts }; @@ -334,14 +294,14 @@ impl OverlayBuilder { anchor_hash: BlockHash, ) -> ProviderResult<(Arc, Arc)> { match &self.overlay_source { - Some(OverlaySource::Managed { manager }) => { + Some(OverlaySource::Managed) => { if anchor_hash == self.parent_hash { Ok(( Arc::new(TrieUpdatesSorted::default()), Arc::new(HashedPostStateSorted::default()), )) } else { - manager + self.overlay_manager .overlay_for_parent(self.parent_hash, anchor_hash) .map_err(ProviderError::other) } @@ -371,9 +331,16 @@ impl OverlayBuilder { let Some(anchor_hash) = self.reused_sparse_trie_anchor_hash else { return false }; match &self.overlay_source { - Some(OverlaySource::Managed { manager }) => { - manager.contains_hash(self.parent_hash, anchor_hash, state_trie_tip_hash) && - manager.contains_hash(self.parent_hash, anchor_hash, finish_tip_hash) + Some(OverlaySource::Managed) => { + self.overlay_manager.contains_hash( + self.parent_hash, + anchor_hash, + state_trie_tip_hash, + ) && self.overlay_manager.contains_hash( + self.parent_hash, + anchor_hash, + finish_tip_hash, + ) } _ => false, } @@ -417,9 +384,17 @@ impl OverlayBuilder { // The database is a hybrid view while the state/trie and Finish frontiers differ. A // manager overlay can use that view directly only when its anchor-to-parent path covers // both frontiers; otherwise the database must first be reverted to the overlay anchor. - if let Some(OverlaySource::Managed { manager }) = &self.overlay_source && - manager.contains_hash(self.parent_hash, anchor_hash, state_trie_tip_block.hash) && - manager.contains_hash(self.parent_hash, anchor_hash, finish_tip_block.hash) + if matches!(&self.overlay_source, Some(OverlaySource::Managed)) && + self.overlay_manager.contains_hash( + self.parent_hash, + anchor_hash, + state_trie_tip_block.hash, + ) && + self.overlay_manager.contains_hash( + self.parent_hash, + anchor_hash, + finish_tip_block.hash, + ) { return Ok(None) } @@ -606,13 +581,10 @@ mod tests { let provider = factory.provider().unwrap(); for (parent_index, expected_ids) in [(3, vec![3, 4]), (4, vec![3, 4, 5])] { - let overlay = OverlayBuilder::::new( - blocks[parent_index].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_overlay_manager(manager.clone()) - .build_overlay(&provider) - .unwrap(); + let overlay = manager + .overlay_builder(blocks[parent_index].recovered_block().hash()) + .build_overlay(&provider) + .unwrap(); assert_eq!( account_keys(&overlay), @@ -637,13 +609,10 @@ mod tests { manager.insert_block(blocks[1].clone()); let provider = factory.provider().unwrap(); - let overlay = OverlayBuilder::::new( - blocks[1].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_overlay_manager(manager) - .build_overlay(&provider) - .unwrap(); + let overlay = manager + .overlay_builder(blocks[1].recovered_block().hash()) + .build_overlay(&provider) + .unwrap(); assert!(overlay.hashed_post_state.is_empty()); assert!(overlay.trie_updates.is_empty()); @@ -656,11 +625,7 @@ mod tests { let manager = OverlayManager::default(); manager.insert_block(blocks[2].clone()); let provider = factory.provider().unwrap(); - let builder = OverlayBuilder::::new( - blocks[2].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_overlay_manager(manager); + let builder = manager.overlay_builder(blocks[2].recovered_block().hash()); let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); let anchor_hash = blocks[1].recovered_block().hash(); let revert_blocks = @@ -689,11 +654,7 @@ mod tests { let manager = OverlayManager::default(); manager.insert_block(blocks[2].clone()); let provider = factory.provider().unwrap(); - let builder = OverlayBuilder::::new( - blocks[2].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_overlay_manager(manager); + let builder = manager.overlay_builder(blocks[2].recovered_block().hash()); let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); let anchor_hash = blocks[1].recovered_block().hash(); let revert_blocks = @@ -722,11 +683,7 @@ mod tests { let manager = OverlayManager::default(); manager.insert_block(blocks[2].clone()); let provider = factory.provider().unwrap(); - let builder = OverlayBuilder::::new( - blocks[2].recovered_block().hash(), - ChangesetCache::new(), - ) - .with_overlay_manager(manager); + let builder = manager.overlay_builder(blocks[2].recovered_block().hash()); let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); let anchor_hash = blocks[1].recovered_block().hash(); let error = builder @@ -747,12 +704,11 @@ mod tests { fn overlay_after_state_trie_frontier_requires_managed_coverage() { let (factory, blocks) = setup_frontiers(1, 3); let provider = factory.provider().unwrap(); - let error = OverlayBuilder::::new( - blocks[3].recovered_block().hash(), - ChangesetCache::new(), - ) - .build_overlay(&provider) - .unwrap_err(); + let error = OverlayManager::::default() + .overlay_builder(blocks[3].recovered_block().hash()) + .with_overlay_source(None) + .build_overlay(&provider) + .unwrap_err(); assert!( error.to_string().contains("is after partial state trie frontier"), @@ -766,8 +722,8 @@ mod tests { let (factory, blocks) = setup_frontiers(1, 3); let provider = factory.provider().unwrap(); let parent_hash = blocks[3].recovered_block().hash(); - let error = OverlayBuilder::::new(parent_hash, ChangesetCache::new()) - .with_overlay_manager(OverlayManager::default()) + let error = OverlayManager::::default() + .overlay_builder(parent_hash) .build_overlay(&provider) .unwrap_err(); @@ -777,8 +733,7 @@ mod tests { #[test] fn managed_overlay_skips_manager_for_persisted_parent() { let parent_hash = B256::with_last_byte(1); - let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) - .with_overlay_manager(OverlayManager::default()); + let builder = OverlayManager::::default().overlay_builder(parent_hash); let (trie, state) = builder.resolve_overlays(parent_hash).unwrap(); assert!(trie.is_empty()); @@ -789,8 +744,7 @@ mod tests { fn managed_overlay_errors_if_parent_is_not_persisted_or_managed() { let parent_hash = B256::with_last_byte(1); let anchor_hash = B256::with_last_byte(2); - let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) - .with_overlay_manager(OverlayManager::default()); + let builder = OverlayManager::::default().overlay_builder(parent_hash); let err = builder.resolve_overlays(anchor_hash).unwrap_err(); @@ -800,8 +754,7 @@ mod tests { #[test] fn managed_overlay_skip_requires_both_frontiers() { let parent_hash = B256::with_last_byte(1); - let builder = OverlayBuilder::::new(parent_hash, ChangesetCache::default()) - .with_overlay_manager(OverlayManager::default()); + let builder = OverlayManager::::default().overlay_builder(parent_hash); assert!(!builder.should_skip_overlay_for_reused_sparse_trie(parent_hash, parent_hash)); let builder = builder.with_skip_overlay_for_reused_sparse_trie(parent_hash); @@ -814,12 +767,9 @@ mod tests { for block in &blocks[2..=4] { manager.insert_block(block.clone()); } - let builder = OverlayBuilder::::new( - blocks[4].recovered_block().hash(), - ChangesetCache::default(), - ) - .with_overlay_manager(manager) - .with_skip_overlay_for_reused_sparse_trie(blocks[1].recovered_block().hash()); + let builder = manager + .overlay_builder(blocks[4].recovered_block().hash()) + .with_skip_overlay_for_reused_sparse_trie(blocks[1].recovered_block().hash()); assert!(builder.should_skip_overlay_for_reused_sparse_trie( blocks[1].recovered_block().hash(), blocks[3].recovered_block().hash(), diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index ed062f06fb5..bc6ace93130 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -62,7 +62,7 @@ use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; /// - Block number exceeds database tip /// - Database access fails /// - Cache retrieval fails -pub fn compute_block_trie_updates( +pub(crate) fn compute_block_trie_updates( cache: &ChangesetCache, provider: &Provider, block_number: BlockNumber, @@ -149,7 +149,7 @@ where /// This type wraps a shared, mutable reference to the cache inner. /// The `RwLock` enables concurrent reads while ensuring exclusive access for writes. #[derive(Debug, Clone)] -pub struct ChangesetCache { +pub(crate) struct ChangesetCache { inner: Arc>, } @@ -164,22 +164,10 @@ impl ChangesetCache { /// /// The cache has no capacity limit and relies on explicit eviction /// via the `evict()` method to manage memory usage. - pub fn new() -> Self { + pub(crate) fn new() -> Self { Self { inner: Arc::new(RwLock::new(ChangesetCacheInner::new())) } } - /// Retrieves changesets for a block. - /// - /// Returns `None` if the block is not in the cache (either evicted or never computed). - /// Updates hit/miss metrics accordingly. - pub fn get( - &self, - block_hash: B256, - block_number: BlockNumber, - ) -> Option> { - self.inner.read().get(&ChangesetRangeKey::single(block_number, block_hash)) - } - /// Evicts changesets for blocks below the given block number. /// /// This should be called after blocks are persisted to the database to free @@ -189,7 +177,7 @@ impl ChangesetCache { /// /// * `up_to_block` - Evict blocks with number < this value. Blocks with number >= this value /// are retained. - pub fn evict(&self, up_to_block: BlockNumber) { + pub(crate) fn evict(&self, up_to_block: BlockNumber) { self.inner.write().evict(up_to_block) } @@ -206,7 +194,7 @@ impl ChangesetCache { /// # Returns /// /// Changesets for the block, either from cache or computed on-the-fly. - pub fn get_or_compute

( + pub(crate) fn get_or_compute

( &self, provider: &P, block_number: BlockNumber, @@ -247,7 +235,7 @@ impl ChangesetCache { /// - Database access fails /// - Block hash lookup fails /// - Changeset computation fails - pub fn get_or_compute_range

( + pub(crate) fn get_or_compute_range

( &self, provider: &P, range: RangeInclusive, diff --git a/crates/storage/storage-overlay/src/lib.rs b/crates/storage/storage-overlay/src/lib.rs index dac37519f25..d97f1fb67c0 100644 --- a/crates/storage/storage-overlay/src/lib.rs +++ b/crates/storage/storage-overlay/src/lib.rs @@ -6,7 +6,7 @@ mod builder; pub use builder::*; mod changeset_cache; -pub use changeset_cache::*; +pub(crate) use changeset_cache::ChangesetCache; mod manager; pub use manager::*; diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 21306ef9d73..f824935a0b7 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -4,9 +4,11 @@ //! parent has not been persisted yet. [`OverlayManager`] tracks those in-memory blocks and //! builds reusable flattened state trie overlays on demand. -use alloy_primitives::B256; +use crate::{changeset_cache::compute_block_trie_updates, ChangesetCache, OverlayBuilder}; +use alloy_primitives::{BlockNumber, B256}; use parking_lot::Mutex; use reth_chain_state::{ExecutedBlock, PreservedSparseTrie}; +use reth_errors::ProviderResult; use reth_ethereum_primitives::EthPrimitives; use reth_metrics::{ metrics::{Counter, Histogram}, @@ -16,11 +18,15 @@ use reth_primitives_traits::{ dashmap::{mapref::entry::Entry, DashMap}, AlloyBlockHeader, FastInstant, NodePrimitives, }; +use reth_storage_api::{ + BlockNumReader, ChangeSetReader, DBProvider, StorageChangeSetReader, StorageSettingsCache, +}; #[cfg(feature = "rayon")] use reth_tasks::WorkerPool; use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted}; use std::{ fmt, + ops::RangeInclusive, sync::{Arc, OnceLock}, time::Instant, }; @@ -28,12 +34,13 @@ use tracing::{debug, trace}; /// Manages flattened state trie overlays for in-memory blocks. /// -/// The manager owns the in-memory block graph and a cache of flattened state trie overlays keyed by -/// `(anchor_hash, tip_hash)`. +/// The manager owns the in-memory block graph, changeset cache, and a cache of flattened state trie +/// overlays keyed by `(anchor_hash, tip_hash)`. #[derive(Clone)] pub struct OverlayManager { blocks: Arc>>, overlays: Arc>, + changeset_cache: ChangesetCache, preserved_sparse_trie: Arc>>, #[cfg(feature = "rayon")] worker_pool: Option>, @@ -57,6 +64,7 @@ impl Default for OverlayManager { Self { blocks: Default::default(), overlays: Default::default(), + changeset_cache: Default::default(), preserved_sparse_trie: Default::default(), #[cfg(feature = "rayon")] worker_pool: None, @@ -81,12 +89,55 @@ impl OverlayManager { Self { blocks: Default::default(), overlays: Default::default(), + changeset_cache: Default::default(), preserved_sparse_trie: Default::default(), worker_pool: Some(worker_pool), metrics: Default::default(), } } + /// Creates an overlay builder for `parent_hash`. + pub fn overlay_builder(&self, parent_hash: B256) -> OverlayBuilder { + OverlayBuilder::new(parent_hash, self.clone()) + } + + /// Gets or computes cached changesets for an inclusive block range. + pub fn get_or_compute_cached_changesets_range

( + &self, + provider: &P, + range: RangeInclusive, + ) -> ProviderResult> + where + P: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, + { + self.changeset_cache.get_or_compute_range(provider, range) + } + + /// Evicts cached changesets for blocks below `up_to_block`. + pub fn evict_cached_changesets(&self, up_to_block: BlockNumber) { + self.changeset_cache.evict(up_to_block); + } + + /// Computes the trie updates produced by `block_number`. + pub fn compute_block_trie_updates

( + &self, + provider: &P, + block_number: BlockNumber, + ) -> ProviderResult + where + P: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + StorageSettingsCache, + { + compute_block_trie_updates(&self.changeset_cache, provider, block_number) + } + /// Takes the preserved sparse trie if present. pub fn take_sparse_trie(&self) -> Option { self.preserved_sparse_trie.lock().take() diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index 06da929ab00..6d508e7e4c2 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -1173,13 +1173,12 @@ mod tests { let chain_spec = Arc::new(ChainSpec::default()); let anchor_hash = chain_spec.genesis_hash(); let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec); - let changeset_cache = reth_storage_overlay::ChangesetCache::new(); let factory = reth_provider::providers::OverlayStateProviderFactory::new( provider_factory, - reth_storage_overlay::OverlayBuilder::::new( - anchor_hash, - changeset_cache, - ), + reth_storage_overlay::OverlayManager::< + reth_ethereum_primitives::EthPrimitives, + >::default() + .overlay_builder(anchor_hash), ); let ctx = test_ctx(factory); diff --git a/examples/custom-state-root/src/main.rs b/examples/custom-state-root/src/main.rs index fc9cad260fc..546d28538af 100644 --- a/examples/custom-state-root/src/main.rs +++ b/examples/custom-state-root/src/main.rs @@ -49,7 +49,7 @@ use reth_ethereum::{ use reth_evm::{revm::context::Block as _, ConfigureEvm}; use reth_primitives_traits::{NodePrimitives, RecoveredBlock}; use reth_provider::{BlockExecutionOutput, ProviderResult}; -use reth_storage_overlay::{ChangesetCache, OverlayManager}; +use reth_storage_overlay::OverlayManager; use reth_trie::updates::TrieUpdates; /// Strategy that returns `B256::ZERO` as the state root from an activation timestamp on, and @@ -148,13 +148,9 @@ where self, ctx: &reth_ethereum::node::builder::AddOnsContext<'_, N>, tree_config: TreeConfig, - changeset_cache: ChangesetCache, - state_trie_overlays: OverlayManager, + overlay_manager: OverlayManager, ) -> eyre::Result { - let validator = self - .inner - .build_tree_validator(ctx, tree_config, changeset_cache, state_trie_overlays) - .await?; + let validator = self.inner.build_tree_validator(ctx, tree_config, overlay_manager).await?; let state_root_strategy: Arc> = self.state_root_strategy; Ok(validator.with_state_root_strategy(state_root_strategy)) From 71dddc81b059293adf4da564390f0f75ec777ceb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 29 Jul 2026 02:27:03 +0200 Subject: [PATCH 059/148] chore(deps): bump the ci-weekly group with 4 updates (#26501) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/bench-benchmarkoor.yml | 2 +- .github/workflows/bench-scheduled.yml | 6 +-- .github/workflows/bench.yml | 2 +- .github/workflows/book.yml | 2 +- .github/workflows/check-alloy.yml | 2 +- .github/workflows/compact.yml | 6 +-- .github/workflows/docker-test.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/e2e.yml | 8 ++-- .github/workflows/fetch-grafana-dashboard.yml | 4 +- .github/workflows/grafana.yml | 2 +- .github/workflows/hive.yml | 14 +++---- .github/workflows/integration.yml | 8 ++-- .github/workflows/kurtosis.yml | 2 +- .github/workflows/label-pr.yml | 2 +- .github/workflows/lint-actions.yml | 2 +- .github/workflows/lint.yml | 38 +++++++++---------- .github/workflows/release-reproducible.yml | 2 +- .github/workflows/release.yml | 6 +-- .github/workflows/reproducible-build.yml | 2 +- .github/workflows/stage.yml | 2 +- .github/workflows/sync-era.yml | 2 +- .github/workflows/sync.yml | 2 +- .github/workflows/unit.yml | 12 +++--- 24 files changed, 66 insertions(+), 66 deletions(-) diff --git a/.github/workflows/bench-benchmarkoor.yml b/.github/workflows/bench-benchmarkoor.yml index 17c70c3682b..e2cdce401b3 100644 --- a/.github/workflows/bench-benchmarkoor.yml +++ b/.github/workflows/bench-benchmarkoor.yml @@ -190,7 +190,7 @@ jobs: lsb_release -a 2>/dev/null || true echo "::endgroup::" - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false submodules: true diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index 1d89cc0f713..5f8d955bdd1 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -89,7 +89,7 @@ jobs: blocks: ${{ steps.bench-config.outputs.blocks }} warmup: ${{ steps.bench-config.outputs.warmup }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: .github/scripts @@ -326,7 +326,7 @@ jobs: - name: Clean up previous bench-work run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false submodules: true @@ -1093,7 +1093,7 @@ jobs: name: save-state runs-on: ubuntu-latest steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false sparse-checkout: .github/scripts diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 207c0fd7c17..f88d69b989a 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -847,7 +847,7 @@ jobs: core.setOutput('repository', repository); core.setOutput('ref', sha); - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ${{ steps.checkout-ref.outputs.repository }} persist-credentials: false diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index 8e6f64469d4..c8cea3e8832 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -20,7 +20,7 @@ jobs: timeout-minutes: 90 steps: - name: Checkout - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/check-alloy.yml b/.github/workflows/check-alloy.yml index e8e7e3b8a4e..64e68867669 100644 --- a/.github/workflows/check-alloy.yml +++ b/.github/workflows/check-alloy.yml @@ -32,7 +32,7 @@ jobs: contents: read timeout-minutes: 60 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 378bcdc67be..86f2592881a 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -29,7 +29,7 @@ jobs: - cargo run --bin reth --features "dev" steps: - name: Checkout PR - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -40,7 +40,7 @@ jobs: with: cache-on-failure: true - name: Checkout base - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ github.base_ref || 'main' }} persist-credentials: false @@ -49,7 +49,7 @@ jobs: run: | ${{ matrix.bin }} -- test-vectors compact --write - name: Checkout PR - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: clean: false persist-credentials: false diff --git a/.github/workflows/docker-test.yml b/.github/workflows/docker-test.yml index 5fe20ad6bdf..91b57deebf7 100644 --- a/.github/workflows/docker-test.yml +++ b/.github/workflows/docker-test.yml @@ -23,7 +23,7 @@ jobs: id-token: write contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - run: mkdir -p artifacts diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cdd6b24737e..5441c81cfd9 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -41,7 +41,7 @@ jobs: contents: read id-token: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5a4d0686233..cd7f3450078 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -29,14 +29,14 @@ jobs: RUST_BACKTRACE: 1 timeout-minutes: 90 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: nextest - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 @@ -64,14 +64,14 @@ jobs: RUST_BACKTRACE: 1 timeout-minutes: 60 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: nextest - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 diff --git a/.github/workflows/fetch-grafana-dashboard.yml b/.github/workflows/fetch-grafana-dashboard.yml index 41f44225f3d..4e7e24714b8 100644 --- a/.github/workflows/fetch-grafana-dashboard.yml +++ b/.github/workflows/fetch-grafana-dashboard.yml @@ -21,11 +21,11 @@ jobs: contents: write pull-requests: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 with: python-version: "3.12" diff --git a/.github/workflows/grafana.yml b/.github/workflows/grafana.yml index 1567997d509..700091f6a4e 100644 --- a/.github/workflows/grafana.yml +++ b/.github/workflows/grafana.yml @@ -14,7 +14,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Validate dashboard format diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index 9bef4675d95..bd5e1d7c7db 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -41,11 +41,11 @@ jobs: - osaka name: Prepare Hive - ${{ matrix.variant == 'amsterdam' && 'Amsterdam' || 'Osaka' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Checkout hive tests - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ethereum/hive path: hivetests @@ -55,7 +55,7 @@ jobs: id: hive-commit run: echo "hash=$(cd hivetests && git rev-parse HEAD)" >> $GITHUB_OUTPUT - - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "^1.13.1" - run: go version @@ -206,7 +206,7 @@ jobs: contents: read issues: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 @@ -232,7 +232,7 @@ jobs: chmod +x /usr/local/bin/hive - name: Checkout hive tests - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ethereum/hive ref: master @@ -385,7 +385,7 @@ jobs: contents: read issues: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 @@ -411,7 +411,7 @@ jobs: chmod +x /usr/local/bin/hive - name: Checkout hive tests - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ethereum/hive ref: master diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3852bf154b0..a0afb73c59d 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -36,7 +36,7 @@ jobs: network: ["ethereum"] timeout-minutes: 60 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -44,7 +44,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - name: Install Geth run: .github/scripts/install_geth.sh - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -79,13 +79,13 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 diff --git a/.github/workflows/kurtosis.yml b/.github/workflows/kurtosis.yml index ed3aa5a0c7b..ff8a910e3f0 100644 --- a/.github/workflows/kurtosis.yml +++ b/.github/workflows/kurtosis.yml @@ -42,7 +42,7 @@ jobs: needs: - build-reth steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 diff --git a/.github/workflows/label-pr.yml b/.github/workflows/label-pr.yml index 4203209f85b..a6b1e2c94a6 100644 --- a/.github/workflows/label-pr.yml +++ b/.github/workflows/label-pr.yml @@ -14,7 +14,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 diff --git a/.github/workflows/lint-actions.yml b/.github/workflows/lint-actions.yml index b2e234022cb..b2374d13b9e 100644 --- a/.github/workflows/lint-actions.yml +++ b/.github/workflows/lint-actions.yml @@ -16,7 +16,7 @@ jobs: permissions: contents: read steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Download actionlint diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 7f85d2913a3..6081c0c2439 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -26,7 +26,7 @@ jobs: args: --workspace --lib --examples --tests --benches --locked features: "asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs" steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -54,7 +54,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -76,14 +76,14 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -102,14 +102,14 @@ jobs: contents: read timeout-minutes: 60 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: dtolnay/rust-toolchain@stable with: target: riscv32imac-unknown-none-elf - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -131,13 +131,13 @@ jobs: total_partitions: [3] timeout-minutes: 60 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -153,7 +153,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -176,7 +176,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -199,7 +199,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -217,7 +217,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -227,7 +227,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: cache-on-failure: true - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: cargo-udeps - run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked @@ -239,7 +239,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -262,7 +262,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: crate-ci/typos@aca895bf05aec0cb7dffa6f94495e923224d9f17 # v1.46.2 @@ -274,7 +274,7 @@ jobs: timeout-minutes: 30 steps: - name: Checkout repository - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Run dprint @@ -288,7 +288,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Check dashboard JSON with jq @@ -302,7 +302,7 @@ jobs: contents: read timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -317,7 +317,7 @@ jobs: contents: read timeout-minutes: 20 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - run: .github/scripts/install_llvm.sh ubuntu diff --git a/.github/workflows/release-reproducible.yml b/.github/workflows/release-reproducible.yml index 9d5e78e6762..93c46014e33 100644 --- a/.github/workflows/release-reproducible.yml +++ b/.github/workflows/release-reproducible.yml @@ -49,7 +49,7 @@ jobs: packages: write contents: write steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ needs.extract-version.outputs.VERSION }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a6ceda45ffd..3e3e6fa753b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -62,7 +62,7 @@ jobs: needs: extract-version if: ${{ github.event.inputs.dry_run != 'true' }} steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ env.RELEASE_REF }} @@ -108,7 +108,7 @@ jobs: - command: build binary: reth steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false ref: ${{ env.RELEASE_REF }} @@ -193,7 +193,7 @@ jobs: steps: # This is necessary for generating the changelog. # It has to come before "Download Artifacts" or else it deletes the artifacts. - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false fetch-depth: 0 diff --git a/.github/workflows/reproducible-build.yml b/.github/workflows/reproducible-build.yml index 70771145da6..5251696c777 100644 --- a/.github/workflows/reproducible-build.yml +++ b/.github/workflows/reproducible-build.yml @@ -22,7 +22,7 @@ jobs: - runner: ubuntu-22.04 machine: machine-2 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 1e1c4aeebda..8d0e2bf9d04 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -33,7 +33,7 @@ jobs: RUST_BACKTRACE: 1 timeout-minutes: 60 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 diff --git a/.github/workflows/sync-era.yml b/.github/workflows/sync-era.yml index 71105cdb983..bbd1ee0b472 100644 --- a/.github/workflows/sync-era.yml +++ b/.github/workflows/sync-era.yml @@ -38,7 +38,7 @@ jobs: block: 100000 unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a" steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 diff --git a/.github/workflows/sync.yml b/.github/workflows/sync.yml index 46b5e7bf5b2..7a2b8c69801 100644 --- a/.github/workflows/sync.yml +++ b/.github/workflows/sync.yml @@ -38,7 +38,7 @@ jobs: block: 100000 unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a" steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 4dfa3c5494d..41274c56cb3 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -36,7 +36,7 @@ jobs: exclude_args: "" timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 @@ -47,7 +47,7 @@ jobs: with: cache-targets: false cache-on-failure: true - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: nextest - if: "${{ matrix.type == 'book' }}" @@ -73,11 +73,11 @@ jobs: RUST_BACKTRACE: 1 timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - name: Checkout ethereum/tests - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: repository: ethereum/tests ref: 81862e4848585a438d64f911a19b3825f0f4cd95 @@ -97,7 +97,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@43aecc8d72668fbcfe75c31400bc4f890f1c5853 # v2.83.2 + - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -115,7 +115,7 @@ jobs: RUST_BACKTRACE: 1 timeout-minutes: 30 steps: - - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 From b813f1fb4b8433e16687385484c34226b241185c Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 29 Jul 2026 14:53:09 +0200 Subject: [PATCH 060/148] refactor(engine): parse payload IDs directly (#26519) --- crates/ethereum/node/src/engine_ssz_proxy.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/crates/ethereum/node/src/engine_ssz_proxy.rs b/crates/ethereum/node/src/engine_ssz_proxy.rs index da1fc821e50..8071df39611 100644 --- a/crates/ethereum/node/src/engine_ssz_proxy.rs +++ b/crates/ethereum/node/src/engine_ssz_proxy.rs @@ -15,7 +15,7 @@ use crate::engine_ssz_containers::{ }; use alloy_consensus::{Transaction, TxEnvelope}; use alloy_eips::{eip2718::Decodable2718, eip7685::RequestsOrHash}; -use alloy_primitives::{Bytes, B128, B256, B64}; +use alloy_primitives::{Bytes, B128, B256}; use alloy_rpc_types_engine::{ CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadFieldV2, ExecutionPayloadSidecar, ForkchoiceState, PayloadAttributes, PayloadId, PraguePayloadFields, @@ -299,7 +299,7 @@ fn parse_engine_path(path: &str) -> Option { Some(EngineSszEndpoint::NewPayload) } (Some("engine"), Some("v1"), Some("payloads"), Some(payload_id), None) => { - let payload_id = payload_id.parse::().map(PayloadId::from); + let payload_id = payload_id.parse::(); Some(EngineSszEndpoint::GetPayload(payload_id)) } (Some("engine"), Some("v1"), Some("forkchoice"), None, None) => { @@ -317,7 +317,7 @@ enum EngineSszEndpoint { Capabilities, Identity, NewPayload, - GetPayload(Result::Err>), + GetPayload(Result::Err>), Forkchoice, Blobs(u8), } From 97a10b1c40749917af7ae3b3312a3a1422ea2912 Mon Sep 17 00:00:00 2001 From: nekomoto911 Date: Wed, 29 Jul 2026 20:56:10 +0800 Subject: [PATCH 061/148] fix(pruner): keep history checkpoint from rewinding on static file walks (#26505) --- .../src/segments/user/account_history.rs | 227 +++++++++++++++++- .../prune/prune/src/segments/user/history.rs | 13 +- .../src/segments/user/storage_history.rs | 156 +++++++++++- 3 files changed, 370 insertions(+), 26 deletions(-) diff --git a/crates/prune/prune/src/segments/user/account_history.rs b/crates/prune/prune/src/segments/user/account_history.rs index af3acb774e8..acd0c312fb0 100644 --- a/crates/prune/prune/src/segments/user/account_history.rs +++ b/crates/prune/prune/src/segments/user/account_history.rs @@ -129,11 +129,17 @@ impl AccountHistory { let walker = StaticFileAccountChangesetWalker::new(provider, range); for result in walker { - if limiter.is_limit_reached() { + let (block_number, changeset) = result?; + // The walk itself deletes nothing, so an interrupted block cannot be resumed: giving + // up the budget inside block N reports checkpoint N-1 and the next run rereads the + // same entries, forever. Stop on block boundaries only, overshooting the budget by at + // most the rest of one block. + if limiter.is_limit_reached() && + last_changeset_pruned_block.is_some_and(|last| last != block_number) + { done = false; break; } - let (block_number, changeset) = result?; highest_deleted_accounts.insert(changeset.address, block_number); last_changeset_pruned_block = Some(block_number); pruned_changesets += 1; @@ -210,9 +216,19 @@ impl AccountHistory { )?; trace!(target: "pruner", pruned = %pruned_changesets, %done, "Pruned account history (changesets from database)"); + // The table walk can stop in the middle of a block, so the interrupted block has to be + // pruned again on the next run. + let last_pruned_block = last_changeset_pruned_block.map(|block_number| { + if done { + block_number + } else { + block_number.saturating_sub(1) + } + }); + let result = HistoryPruneResult { highest_deleted: highest_deleted_accounts, - last_pruned_block: last_changeset_pruned_block, + last_pruned_block, pruned_count: pruned_changesets, done, }; @@ -263,11 +279,17 @@ impl AccountHistory { // to determine which history shard entries need pruning. let walker = StaticFileAccountChangesetWalker::new(provider, range); for result in walker { - if limiter.is_limit_reached() { + let (block_number, changeset) = result?; + // Static file changesets are not deleted here, so an interrupted block cannot be + // resumed: giving up the budget inside block N reports checkpoint N-1 and the next run + // rereads the same entries, forever. Stop on block boundaries only, overshooting the + // budget by at most the rest of one block. + if limiter.is_limit_reached() && + last_changeset_pruned_block.is_some_and(|last| last != block_number) + { done = false; break; } - let (block_number, changeset) = result?; highest_deleted_accounts.insert(changeset.address, block_number); last_changeset_pruned_block = Some(block_number); changesets_processed += 1; @@ -275,9 +297,7 @@ impl AccountHistory { } trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned account changesets from static files"); - let last_changeset_pruned_block = last_changeset_pruned_block - .map(|block_number| if done { block_number } else { block_number.saturating_sub(1) }) - .unwrap_or(range_end); + let last_changeset_pruned_block = last_changeset_pruned_block.unwrap_or(range_end); // Prune RocksDB history shards for affected accounts let mut deleted_shards = 0usize; @@ -762,4 +782,195 @@ mod tests { let final_changesets = db.table::().unwrap(); assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned"); } + + /// A block holding at least a whole run's budget of changesets must not stall the `RocksDB` + /// path: the walk deletes no changesets, so a checkpoint rewound below such a block would make + /// every later run reread it and never advance. + #[test] + fn dense_block_advances_rocksdb_checkpoint() { + use reth_db_api::models::ShardedKey; + use reth_provider::RocksDBProviderFactory; + + let db = TestStageDB::default(); + let mut rng = generators::rng(); + + let blocks = random_block_range( + &mut rng, + 0..=20, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, + ); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + + let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::>(); + let (changesets, _) = random_changeset_range( + &mut rng, + blocks.iter(), + accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))), + 0..0, + 0..0, + ); + // `random_changeset_range` emits exactly 2 account changesets per block (sender + + // recipient), so a budget of 2 makes every block "dense". + assert!(changesets.iter().all(|changeset| changeset.len() == 2)); + + db.insert_changesets_to_static_files(changesets.clone(), None) + .expect("insert changesets to static files"); + + // History index lives in RocksDB on the v2 path. + let mut account_blocks: BTreeMap<_, Vec> = BTreeMap::new(); + for (block, changeset) in changesets.iter().enumerate() { + for (address, _, _) in changeset { + account_blocks.entry(*address).or_default().push(block as u64); + } + } + let rocksdb = db.factory.rocksdb_provider(); + let mut batch = rocksdb.batch(); + for (address, block_numbers) in &account_blocks { + let shard = BlockNumberList::new_pre_sorted(block_numbers.iter().copied()); + batch + .put::(ShardedKey::new(*address, u64::MAX), &shard) + .unwrap(); + } + batch.commit().unwrap(); + + db.factory.set_storage_settings_cache(StorageSettings::v2()); + + let to_block: BlockNumber = 15; + let prune_mode = PruneMode::Before(to_block); + let segment = AccountHistory::new(prune_mode); + + // Start from a checkpoint in the middle so a rewind can't be masked by block 0. + let mut checkpoint = PruneCheckpoint { block_number: Some(4), tx_number: None, prune_mode }; + + let run_prune = |checkpoint: PruneCheckpoint, limit: usize| { + let input = PruneInput { + previous_checkpoint: Some(checkpoint), + to_block, + limiter: PruneLimiter::default().set_deleted_entries_limit(limit), + }; + + let provider = db.factory.database_provider_rw().unwrap(); + provider.set_storage_settings_cache(StorageSettings::v2()); + let result = segment.prune(&provider, input).unwrap(); + segment + .save_checkpoint( + &provider, + result.checkpoint.unwrap().as_prune_checkpoint(prune_mode), + ) + .unwrap(); + provider.commit().expect("commit"); + + let checkpoint = db + .factory + .provider() + .unwrap() + .get_prune_checkpoint(PruneSegment::AccountHistory) + .unwrap() + .unwrap(); + (result, checkpoint) + }; + + // The RocksDB path does not halve the limit, so a budget of 2 is exactly one dense block. + for _ in 0..3 { + let previous = checkpoint.block_number; + let (result, next) = run_prune(checkpoint, 2); + checkpoint = next; + + assert!( + !result.progress.is_finished(), + "the range is longer than one run's budget allows" + ); + assert!( + checkpoint.block_number > previous, + "checkpoint must advance past the dense block, got {:?} after {previous:?}", + checkpoint.block_number + ); + } + assert_eq!(checkpoint.block_number, Some(7), "one dense block cleared per run"); + + // With enough budget the remainder of the range completes in one run. + let (result, checkpoint) = run_prune(checkpoint, 1000); + assert!(result.progress.is_finished()); + assert_eq!(checkpoint.block_number, Some(to_block)); + } + + /// Same guarantee for `prune_static_files`, which shares the changeset walk. `Segment::prune` + /// cannot reach it while `storage_v2` short-circuits to the `RocksDB` path, so it is called + /// directly. + #[test] + fn dense_block_advances_static_file_checkpoint() { + let db = TestStageDB::default(); + let mut rng = generators::rng(); + + let blocks = random_block_range( + &mut rng, + 0..=20, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, + ); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + + let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::>(); + let (changesets, _) = random_changeset_range( + &mut rng, + blocks.iter(), + accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))), + 0..0, + 0..0, + ); + assert!(changesets.iter().all(|changeset| changeset.len() == 2)); + + // Changesets in static files. `StaticFileAccountChangesetWalker` resolves entries through + // `ChangeSetReader`, which only reads static files when `storage_v2` is set, so the walk + // needs v2 settings even though v2 routes `prune()` to the RocksDB path. + db.insert_changesets_to_static_files(changesets, None) + .expect("insert changesets to static files"); + db.factory.set_storage_settings_cache(StorageSettings::v2()); + assert!(db.table::().unwrap().is_empty()); + + let to_block: BlockNumber = 15; + let prune_mode = PruneMode::Before(to_block); + let segment = AccountHistory::new(prune_mode); + + let mut checkpoint = PruneCheckpoint { block_number: Some(4), tx_number: None, prune_mode }; + + for _ in 0..3 { + let previous = checkpoint.block_number; + let input = PruneInput { + previous_checkpoint: Some(checkpoint), + to_block, + // Halved internally by ACCOUNT_HISTORY_TABLES_TO_PRUNE, so 4 == one dense block. + limiter: PruneLimiter::default() + .set_deleted_entries_limit(2 * ACCOUNT_HISTORY_TABLES_TO_PRUNE), + }; + let range = input.get_next_block_range().unwrap(); + let range_end = *range.end(); + + let provider = db.factory.database_provider_rw().unwrap(); + provider.set_storage_settings_cache(StorageSettings::v2()); + let result = segment.prune_static_files(&provider, input, range, range_end).unwrap(); + segment + .save_checkpoint( + &provider, + result.checkpoint.unwrap().as_prune_checkpoint(prune_mode), + ) + .unwrap(); + provider.commit().expect("commit"); + + checkpoint = db + .factory + .provider() + .unwrap() + .get_prune_checkpoint(PruneSegment::AccountHistory) + .unwrap() + .unwrap(); + + assert!(!result.progress.is_finished()); + assert!( + checkpoint.block_number > previous, + "checkpoint must advance past the dense block, got {:?} after {previous:?}", + checkpoint.block_number + ); + } + assert_eq!(checkpoint.block_number, Some(7), "one dense block cleared per run"); + } } diff --git a/crates/prune/prune/src/segments/user/history.rs b/crates/prune/prune/src/segments/user/history.rs index d4e6ddcf78d..3211d082700 100644 --- a/crates/prune/prune/src/segments/user/history.rs +++ b/crates/prune/prune/src/segments/user/history.rs @@ -29,7 +29,10 @@ pub(crate) struct PrunedIndices { pub(crate) struct HistoryPruneResult { /// Map of the highest deleted changeset keys to their block numbers. pub(crate) highest_deleted: FxHashMap, - /// The last block number that had changesets pruned. + /// The highest block number whose changesets are fully pruned, becoming the checkpoint. + /// + /// Checkpoints have block granularity, so a caller that can stop in the middle of a block + /// must report the block before it, and prune the interrupted block again on the next run. pub(crate) last_pruned_block: Option, /// Number of changesets pruned. pub(crate) pruned_count: usize, @@ -56,11 +59,9 @@ where { let HistoryPruneResult { highest_deleted, last_pruned_block, pruned_count, done } = result; - // If there's more changesets to prune, set the checkpoint block number to previous, - // so we could finish pruning its changesets on the next run. - let last_changeset_pruned_block = last_pruned_block - .map(|block_number| if done { block_number } else { block_number.saturating_sub(1) }) - .unwrap_or(range_end); + // Nothing was pruned only when the range held no changesets at all, so the whole range is + // done. + let last_changeset_pruned_block = last_pruned_block.unwrap_or(range_end); // Sort highest deleted block numbers and turn them into sharded keys. // We use `sorted_unstable` because no equal keys exist in the map. diff --git a/crates/prune/prune/src/segments/user/storage_history.rs b/crates/prune/prune/src/segments/user/storage_history.rs index c475b7e61d9..06a8522ea39 100644 --- a/crates/prune/prune/src/segments/user/storage_history.rs +++ b/crates/prune/prune/src/segments/user/storage_history.rs @@ -127,13 +127,19 @@ impl StorageHistory { let walker = provider.static_file_provider().walk_storage_changeset_range(range); for result in walker { - if limiter.is_limit_reached() { - done = false; - break; - } let (block_address, entry) = result?; let block_number = block_address.block_number(); let address = block_address.address(); + // The walk itself deletes nothing, so an interrupted block cannot be resumed: giving + // up the budget inside block N reports checkpoint N-1 and the next run rereads the + // same entries, forever. Stop on block boundaries only, overshooting the budget by at + // most the rest of one block. + if limiter.is_limit_reached() && + last_changeset_pruned_block.is_some_and(|last| last != block_number) + { + done = false; + break; + } highest_deleted_storages.insert((address, entry.key), block_number); last_changeset_pruned_block = Some(block_number); pruned_changesets += 1; @@ -212,9 +218,19 @@ impl StorageHistory { )?; trace!(target: "pruner", deleted = %pruned_changesets, %done, "Pruned storage history (changesets)"); + // The table walk can stop in the middle of a block, so the interrupted block has to be + // pruned again on the next run. + let last_pruned_block = last_changeset_pruned_block.map(|block_number| { + if done { + block_number + } else { + block_number.saturating_sub(1) + } + }); + let result = HistoryPruneResult { highest_deleted: highest_deleted_storages, - last_pruned_block: last_changeset_pruned_block, + last_pruned_block, pruned_count: pruned_changesets, done, }; @@ -264,13 +280,19 @@ impl StorageHistory { // pair to determine which history shard entries need pruning. let walker = provider.static_file_provider().walk_storage_changeset_range(range); for result in walker { - if limiter.is_limit_reached() { - done = false; - break; - } let (block_address, entry) = result?; let block_number = block_address.block_number(); let address = block_address.address(); + // Static file changesets are not deleted here, so an interrupted block cannot be + // resumed: giving up the budget inside block N reports checkpoint N-1 and the next run + // rereads the same entries, forever. Stop on block boundaries only, overshooting the + // budget by at most the rest of one block. + if limiter.is_limit_reached() && + last_changeset_pruned_block.is_some_and(|last| last != block_number) + { + done = false; + break; + } highest_deleted_storages.insert((address, entry.key), block_number); last_changeset_pruned_block = Some(block_number); changesets_processed += 1; @@ -279,9 +301,7 @@ impl StorageHistory { trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned storage changesets from static files"); - let last_changeset_pruned_block = last_changeset_pruned_block - .map(|block_number| if done { block_number } else { block_number.saturating_sub(1) }) - .unwrap_or(range_end); + let last_changeset_pruned_block = last_changeset_pruned_block.unwrap_or(range_end); // Prune RocksDB history shards for affected storage slots let mut deleted_shards = 0usize; @@ -772,4 +792,116 @@ mod tests { } } } + + /// A block holding at least a whole run's budget of changesets must not stall pruning: the + /// walk deletes no changesets, so a checkpoint rewound below such a block would make every + /// later run reread it and never advance. + #[test] + fn dense_block_advances_rocksdb_checkpoint() { + use alloy_primitives::U256; + use reth_db_api::models::storage_sharded_key::StorageShardedKey; + use reth_primitives_traits::StorageEntry; + use reth_provider::RocksDBProviderFactory; + use reth_storage_api::StorageSettings; + + let db = TestStageDB::default(); + let mut rng = generators::rng(); + + let blocks = random_block_range( + &mut rng, + 0..=20, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, + ); + db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks"); + + // Two storage changesets per block, so a budget of two makes every block "dense". + const ENTRIES_PER_BLOCK: usize = 2; + let (address, account) = random_eoa_accounts(&mut rng, 1).into_iter().next().unwrap(); + let keys = [B256::with_last_byte(1), B256::with_last_byte(2)]; + let changesets = (0..=20) + .map(|_| { + vec![( + address, + account, + keys.iter() + .map(|key| StorageEntry { key: *key, value: U256::from(1) }) + .collect(), + )] + }) + .collect::>(); + db.insert_changesets_to_static_files(changesets, None) + .expect("insert changesets to static files"); + + { + let rocksdb = db.factory.rocksdb_provider(); + let mut batch = rocksdb.batch(); + for key in keys { + batch + .put::( + StorageShardedKey::last(address, key), + &BlockNumberList::new_pre_sorted(0..=20), + ) + .expect("insert storage history shard"); + } + batch.commit().expect("commit rocksdb batch"); + } + + let to_block = 15u64; + let prune_mode = PruneMode::Before(to_block); + let segment = StorageHistory::new(prune_mode); + + // Start from a checkpoint in the middle so a rewind can't be masked by block 0. + let mut checkpoint = PruneCheckpoint { block_number: Some(4), tx_number: None, prune_mode }; + + let run_prune = |checkpoint: PruneCheckpoint, limit: usize| { + let input = PruneInput { + previous_checkpoint: Some(checkpoint), + to_block, + limiter: PruneLimiter::default().set_deleted_entries_limit(limit), + }; + + let provider = db.factory.database_provider_rw().unwrap(); + provider.set_storage_settings_cache(StorageSettings::v2()); + let result = segment.prune(&provider, input).unwrap(); + segment + .save_checkpoint( + &provider, + result.checkpoint.unwrap().as_prune_checkpoint(prune_mode), + ) + .unwrap(); + provider.commit().expect("commit"); + + let checkpoint = db + .factory + .provider() + .unwrap() + .get_prune_checkpoint(PruneSegment::StorageHistory) + .unwrap() + .unwrap(); + (result, checkpoint) + }; + + // The RocksDB path does not halve the limit, so this budget is exactly one dense block. + for _ in 0..3 { + let previous = checkpoint.block_number; + let (result, next) = run_prune(checkpoint, ENTRIES_PER_BLOCK); + checkpoint = next; + + assert!( + !result.progress.is_finished(), + "the range is longer than one run's budget allows" + ); + assert!( + checkpoint.block_number > previous, + "checkpoint must advance past the dense block, got {:?} after {previous:?}", + checkpoint.block_number + ); + } + assert_eq!(checkpoint.block_number, Some(7), "one dense block cleared per run"); + + // With enough budget the remainder of the range completes in one run. + let (result, checkpoint) = run_prune(checkpoint, 1000); + assert!(result.progress.is_finished()); + assert_eq!(checkpoint.block_number, Some(to_block)); + } } From ce463098f20fb335b74d306d81c3d1f661693832 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 29 Jul 2026 16:05:03 +0200 Subject: [PATCH 062/148] refactor(rpc): use Alloy gas limit validation (#26518) --- crates/rpc/rpc/src/validation.rs | 40 +++++++------------------------- 1 file changed, 9 insertions(+), 31 deletions(-) diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index e581fdda53a..e127a4d9028 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -31,8 +31,7 @@ use reth_metrics::{ }; use reth_node_api::{NewPayloadError, PayloadTypes}; use reth_primitives_traits::{ - constants::GAS_LIMIT_BOUND_DIVISOR, BlockBody, GotExpected, NodePrimitives, RecoveredBlock, - SealedBlock, SealedHeaderFor, + BlockBody, GotExpected, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeaderFor, }; use reth_revm::{cached::CachedReads, database::StateProviderDatabase}; use reth_rpc_api::BlockSubmissionValidationApiServer; @@ -174,7 +173,14 @@ where }; self.consensus.validate_header_against_parent(block.sealed_header(), &parent_header)?; - self.validate_gas_limit(registered_gas_limit, &parent_header, block.sealed_header())?; + parent_header.validate_gas_limit(registered_gas_limit, block.gas_limit()).map_err( + |err| { + ValidationApiError::GasLimitMismatch(GotExpected { + got: err.got, + expected: err.expected, + }) + }, + )?; // Ensure the submitted block access list does not exceed the block gas limit (EIP-7928) if let Some(decoded_bal) = decoded_bal { @@ -273,34 +279,6 @@ where } } - /// Ensures that the chosen gas limit is the closest possible value for the validator's - /// registered gas limit. - /// - /// Ref: - fn validate_gas_limit( - &self, - registered_gas_limit: u64, - parent_header: &SealedHeaderFor, - header: &SealedHeaderFor, - ) -> Result<(), ValidationApiError> { - let max_gas_limit = - parent_header.gas_limit() + parent_header.gas_limit() / GAS_LIMIT_BOUND_DIVISOR - 1; - let min_gas_limit = - parent_header.gas_limit() - parent_header.gas_limit() / GAS_LIMIT_BOUND_DIVISOR + 1; - - let best_gas_limit = - std::cmp::max(min_gas_limit, std::cmp::min(max_gas_limit, registered_gas_limit)); - - if best_gas_limit != header.gas_limit() { - return Err(ValidationApiError::GasLimitMismatch(GotExpected { - got: header.gas_limit(), - expected: best_gas_limit, - })) - } - - Ok(()) - } - /// Ensures that the proposer has received [`BidTrace::value`] for this block. /// /// Firstly attempts to verify the payment by checking the state changes, otherwise falls back From c505ed2e097b142040da1bb36d60472ee56c66c0 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 29 Jul 2026 09:13:50 -0500 Subject: [PATCH 063/148] feat(net): accept discv5 ENR bootnodes via CLI (#26448) Co-authored-by: Matthias Seitz --- crates/net/peers/src/trusted_peer.rs | 29 ++++++++++++++++++++ crates/node/core/Cargo.toml | 2 +- crates/node/core/src/args/network.rs | 13 +++++++-- docs/vocs/docs/pages/cli/reth/node.mdx | 4 +-- docs/vocs/docs/pages/cli/reth/p2p/body.mdx | 4 +-- docs/vocs/docs/pages/cli/reth/p2p/header.mdx | 4 +-- docs/vocs/docs/pages/cli/reth/stage/run.mdx | 4 +-- 7 files changed, 49 insertions(+), 11 deletions(-) diff --git a/crates/net/peers/src/trusted_peer.rs b/crates/net/peers/src/trusted_peer.rs index 049085ecb09..15cc3676e9b 100644 --- a/crates/net/peers/src/trusted_peer.rs +++ b/crates/net/peers/src/trusted_peer.rs @@ -116,6 +116,18 @@ impl FromStr for TrustedPeer { type Err = NodeRecordParseError; fn from_str(s: &str) -> Result { + #[cfg(feature = "secp256k1")] + if s.starts_with("enr:") { + let enr = enr::Enr::::from_str(s) + .map_err(NodeRecordParseError::InvalidUrl)?; + let mut record = NodeRecord::try_from(&enr)?; + if record.tcp_port == 0 { + // Discovery-only ENRs commonly share their UDP port with the RLPx listener. + record.tcp_port = record.udp_port; + } + return Ok(record.into()) + } + use url::Url; // Parse the URL with enode prefix replaced with http. @@ -179,6 +191,23 @@ mod tests { }) } + #[cfg(feature = "secp256k1")] + #[test] + fn test_enr_parse() { + let enr = "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; + let node: TrustedPeer = enr.parse().unwrap(); + + assert_eq!(node, TrustedPeer { + host: Host::Ipv4([127, 0, 0, 1].into()), + tcp_port: 30303, + udp_port: 30303, + id: "ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31387574077f301b421bc84df7266c44e9e6d569fc56be00812904767bf5ccd1fc7f" + .parse() + .unwrap(), + }); + assert!("enr:garbage".parse::().is_err()); + } + #[test] fn test_node_display() { let url = "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@10.3.58.6:30303"; diff --git a/crates/node/core/Cargo.toml b/crates/node/core/Cargo.toml index b0aa9313e16..39caeec04e2 100644 --- a/crates/node/core/Cargo.toml +++ b/crates/node/core/Cargo.toml @@ -32,7 +32,7 @@ reth-discv4.workspace = true reth-discv5.workspace = true reth-net-nat.workspace = true reth-net-banlist.workspace = true -reth-network-peers.workspace = true +reth-network-peers = { workspace = true, features = ["secp256k1"] } reth-prune-types.workspace = true reth-stages-types.workspace = true reth-ethereum-forks.workspace = true diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index f6a601cc7f4..d45bebe6859 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -242,7 +242,7 @@ pub struct NetworkArgs { pub discovery: DiscoveryArgs, #[expect(clippy::doc_markdown)] - /// Comma separated enode URLs of trusted peers for P2P connections. + /// Comma separated enode URLs or ENRs of trusted peers for P2P connections. /// /// --trusted-peers enode://abcd@192.168.0.1:30303 #[arg(long, value_delimiter = ',')] @@ -252,7 +252,7 @@ pub struct NetworkArgs { #[arg(long)] pub trusted_only: bool, - /// Comma separated enode URLs for P2P discovery bootstrap. + /// Comma separated enode URLs or ENRs for P2P discovery bootstrap. /// /// Will fall back to a network-specific default if not specified. #[arg(long, value_delimiter = ',')] @@ -1225,6 +1225,15 @@ mod tests { ); } + #[test] + fn parse_enr_bootnode_args() { + let enr = "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; + let args = CommandParser::::parse_from(["reth", "--bootnodes", enr]).args; + let trusted = + CommandParser::::parse_from(["reth", "--trusted-peers", enr]).args; + assert_eq!(args.bootnodes, Some(trusted.trusted_peers)); + } + #[test] fn parse_retry_strategy_args() { let tests = vec![0, 10]; diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index 23d46efbdb7..fe6645912bb 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -139,7 +139,7 @@ Networking: [default: 200] --trusted-peers - Comma separated enode URLs of trusted peers for P2P connections. + Comma separated enode URLs or ENRs of trusted peers for P2P connections. --trusted-peers enode://abcd@192.168.0.1:30303 @@ -147,7 +147,7 @@ Networking: Connect to or accept from trusted peers only --bootnodes - Comma separated enode URLs for P2P discovery bootstrap. + Comma separated enode URLs or ENRs for P2P discovery bootstrap. Will fall back to a network-specific default if not specified. diff --git a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx index a149443834e..400ffe78c88 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/body.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/body.mdx @@ -79,7 +79,7 @@ Networking: [default: 200] --trusted-peers - Comma separated enode URLs of trusted peers for P2P connections. + Comma separated enode URLs or ENRs of trusted peers for P2P connections. --trusted-peers enode://abcd@192.168.0.1:30303 @@ -87,7 +87,7 @@ Networking: Connect to or accept from trusted peers only --bootnodes - Comma separated enode URLs for P2P discovery bootstrap. + Comma separated enode URLs or ENRs for P2P discovery bootstrap. Will fall back to a network-specific default if not specified. diff --git a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx index fcdfbed536c..673dcca6b6b 100644 --- a/docs/vocs/docs/pages/cli/reth/p2p/header.mdx +++ b/docs/vocs/docs/pages/cli/reth/p2p/header.mdx @@ -79,7 +79,7 @@ Networking: [default: 200] --trusted-peers - Comma separated enode URLs of trusted peers for P2P connections. + Comma separated enode URLs or ENRs of trusted peers for P2P connections. --trusted-peers enode://abcd@192.168.0.1:30303 @@ -87,7 +87,7 @@ Networking: Connect to or accept from trusted peers only --bootnodes - Comma separated enode URLs for P2P discovery bootstrap. + Comma separated enode URLs or ENRs for P2P discovery bootstrap. Will fall back to a network-specific default if not specified. diff --git a/docs/vocs/docs/pages/cli/reth/stage/run.mdx b/docs/vocs/docs/pages/cli/reth/stage/run.mdx index 5997110158d..2e6846de20d 100644 --- a/docs/vocs/docs/pages/cli/reth/stage/run.mdx +++ b/docs/vocs/docs/pages/cli/reth/stage/run.mdx @@ -238,7 +238,7 @@ Networking: [default: 200] --trusted-peers - Comma separated enode URLs of trusted peers for P2P connections. + Comma separated enode URLs or ENRs of trusted peers for P2P connections. --trusted-peers enode://abcd@192.168.0.1:30303 @@ -246,7 +246,7 @@ Networking: Connect to or accept from trusted peers only --bootnodes - Comma separated enode URLs for P2P discovery bootstrap. + Comma separated enode URLs or ENRs for P2P discovery bootstrap. Will fall back to a network-specific default if not specified. From 8617e42f916e496b57552b694d5da27af534bd1c Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 30 Jul 2026 12:13:32 +0200 Subject: [PATCH 064/148] fix(trie): avoid marking empty storage as deleted (#26526) --- crates/trie/db/tests/trie.rs | 64 +++++++++++++++++++++++++++++++++++- crates/trie/trie/src/trie.rs | 9 +++-- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/crates/trie/db/tests/trie.rs b/crates/trie/db/tests/trie.rs index 936c771a02a..23a00203ffd 100644 --- a/crates/trie/db/tests/trie.rs +++ b/crates/trie/db/tests/trie.rs @@ -205,8 +205,70 @@ fn test_empty_storage_root() { let tx = factory.provider_rw().unwrap(); reth_trie_db::with_adapter!(tx, |A| { - let got = DbStorageRoot::<_, A>::from_tx(tx.tx_ref(), address).root().unwrap(); + let (got, walked, updates) = + DbStorageRoot::<_, A>::from_tx(tx.tx_ref(), address).root_with_updates().unwrap(); assert_eq!(got, EMPTY_ROOT_HASH); + assert_eq!(walked, 0); + assert_eq!(updates, StorageTrieUpdates::default()); + }); +} + +#[test] +fn cleared_storage_emits_node_removals_without_deleted_flag() { + let factory = create_test_provider_factory(); + let tx = factory.provider_rw().unwrap(); + let hashed_address = B256::random(); + let hashed_slots = [ + hex!("30af561000000000000000000000000000000000000000000000000000000000"), + hex!("30af569000000000000000000000000000000000000000000000000000000000"), + hex!("30af650000000000000000000000000000000000000000000000000000000000"), + hex!("30af6f0000000000000000000000000000000000000000000000000000000000"), + hex!("30af8f0000000000000000000000000000000000000000000000000000000000"), + hex!("3100000000000000000000000000000000000000000000000000000000000000"), + ] + .map(B256::new); + + for hashed_slot in hashed_slots { + tx.tx_ref() + .put::( + hashed_address, + StorageEntry { key: hashed_slot, value: U256::ONE }, + ) + .unwrap(); + } + + let (_, _, initial_updates) = reth_trie_db::with_adapter!(tx, |A| { + DbStorageRoot::<_, A>::from_tx_hashed(tx.tx_ref(), hashed_address) + .root_with_updates() + .unwrap() + }); + assert!(!initial_updates.storage_nodes_ref().is_empty()); + tx.write_storage_trie_updates_sorted(core::iter::once(( + &hashed_address, + &initial_updates.into_sorted(), + ))) + .unwrap(); + + let mut hashed_storage_cursor = + tx.tx_ref().cursor_dup_write::().unwrap(); + let mut prefix_set = PrefixSetMut::default(); + for hashed_slot in hashed_slots { + let entry = + hashed_storage_cursor.seek_by_key_subkey(hashed_address, hashed_slot).unwrap().unwrap(); + assert_eq!(entry.key, hashed_slot); + hashed_storage_cursor.delete_current().unwrap(); + prefix_set.insert(Nibbles::unpack(hashed_slot)); + } + drop(hashed_storage_cursor); + + reth_trie_db::with_adapter!(tx, |A| { + let (root, _, updates) = DbStorageRoot::<_, A>::from_tx_hashed(tx.tx_ref(), hashed_address) + .with_prefix_set(prefix_set.freeze()) + .root_with_updates() + .unwrap(); + assert_eq!(root, EMPTY_ROOT_HASH); + assert!(!updates.is_deleted()); + assert!(!updates.removed_nodes_ref().is_empty()); }); } diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index ee551db87fd..7bc4e83d011 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -736,13 +736,16 @@ where } = calculation; hashed_storage_cursor.set_hashed_address(hashed_address); - // short circuit on empty storage - if hashed_storage_cursor.is_storage_empty()? { + // Empty storage only needs to be walked when changed prefixes must produce trie updates. + if previous_state.is_none() && + (!retain_updates || prefix_set.is_empty()) && + hashed_storage_cursor.is_storage_empty()? + { Span::current().record("storage_root", format!("{EMPTY_ROOT_HASH:?}")); return Ok(StorageRootProgress::Complete( EMPTY_ROOT_HASH, 0, - StorageTrieUpdates::deleted(), + StorageTrieUpdates::default(), )) } From 95909e8431d4e538551a26d5b8b1317638620cf0 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 30 Jul 2026 12:25:56 +0200 Subject: [PATCH 065/148] feat(provider): support split-frontier save_blocks (#26521) --- crates/engine/tree/src/persistence.rs | 145 ++++--- crates/engine/tree/src/tree/mod.rs | 19 +- crates/engine/tree/src/tree/tests.rs | 21 +- crates/storage/provider/src/lib.rs | 2 +- .../src/providers/blockchain_provider.rs | 11 +- .../provider/src/providers/database/mod.rs | 7 +- .../src/providers/database/provider.rs | 364 ++++++++++++++++-- .../src/providers/database/save_blocks.rs | 213 ++++++++++ crates/trie/common/src/hashed_state.rs | 303 ++++++++++++++- crates/trie/common/src/updates.rs | 358 ++++++++++++++++- crates/trie/common/src/utils.rs | 132 +++++++ 11 files changed, 1440 insertions(+), 135 deletions(-) create mode 100644 crates/storage/provider/src/providers/database/save_blocks.rs diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 3f53119df84..50ce806a77f 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -1,13 +1,12 @@ use crate::metrics::PersistenceMetrics; use alloy_eips::BlockNumHash; use crossbeam_channel::Sender as CrossbeamSender; -use reth_chain_state::ExecutedBlock; use reth_errors::ProviderError; use reth_ethereum_primitives::EthPrimitives; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_provider::{ providers::ProviderNodeTypes, BalProvider, BlockExecutionWriter, BlockHashReader, - ChainStateBlockWriter, DBProvider, DatabaseProviderFactory, ProviderFactory, SaveBlocksMode, + ChainStateBlockWriter, DBProvider, DatabaseProviderFactory, ProviderFactory, SaveBlocksInput, }; use reth_prune::{PrunerError, PrunerWithFactory}; use reth_stages_api::{MetricEvent, MetricEventsSender}; @@ -145,14 +144,14 @@ where Ok(new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num })) } - #[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = blocks.len()))] + #[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = input.persist_rest_blocks().len()))] fn on_save_blocks( &mut self, - blocks: Vec>, + input: SaveBlocksInput, ) -> Result { - let first_block = blocks.first().map(|b| b.recovered_block.num_hash()); - let last_block = blocks.last().map(|b| b.recovered_block.num_hash()); - let block_count = blocks.len(); + let first_block = input.first_persist_rest_block().recovered_block().num_hash(); + let last_block = input.last_block(); + let block_count = input.persist_rest_blocks().len(); let pending_finalized = self.pending_finalized_block.take(); let pending_safe = self.pending_safe_block.take(); @@ -161,35 +160,33 @@ where let start_time = Instant::now(); - if let Some(last) = last_block { - let provider_rw = self.provider.database_provider_rw()?; - provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?; + let provider_rw = self.provider.database_provider_rw()?; + provider_rw.save_blocks(&input)?; - if let Some(finalized) = pending_finalized { - provider_rw.save_finalized_block_number(finalized.min(last.number))?; - if finalized > last.number { - self.pending_finalized_block = Some(finalized); - } + if let Some(finalized) = pending_finalized { + provider_rw.save_finalized_block_number(finalized.min(last_block.number))?; + if finalized > last_block.number { + self.pending_finalized_block = Some(finalized); } - if let Some(safe) = pending_safe { - provider_rw.save_safe_block_number(safe.min(last.number))?; - if safe > last.number { - self.pending_safe_block = Some(safe); - } + } + if let Some(safe) = pending_safe { + provider_rw.save_safe_block_number(safe.min(last_block.number))?; + if safe > last_block.number { + self.pending_safe_block = Some(safe); } - - provider_rw.commit()?; - let _ = self.provider.bal_store().flush().inspect_err(|err| { - warn!(target: "engine::persistence", last=?last_block, ?err, "Failed to flush BAL store"); - }); - debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks"); } + provider_rw.commit()?; + let _ = self.provider.bal_store().flush().inspect_err(|err| { + warn!(target: "engine::persistence", last=?last_block, ?err, "Failed to flush BAL store"); + }); + debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks"); + let elapsed = start_time.elapsed(); self.metrics.save_blocks_batch_size.record(block_count as f64); self.metrics.save_blocks_duration_seconds.record(elapsed); - Ok(PersistenceResult { last_block, commit_duration: Some(elapsed) }) + Ok(PersistenceResult { last_block: Some(last_block), commit_duration: Some(elapsed) }) } fn maybe_run_pruner(&mut self, block_number: u64) -> Result<(), PersistenceError> { @@ -232,12 +229,8 @@ pub enum PersistenceError { /// A signal to the persistence service that part of the tree state can be persisted. #[derive(Debug)] pub enum PersistenceAction { - /// The section of tree state that should be persisted. These blocks are expected in order of - /// increasing block number. - /// - /// First, header, transaction, and receipt-related data should be written to static files. - /// Then the execution history-related data will be written to the database. - SaveBlocks(Vec>, CrossbeamSender), + /// Advances the block-data and state/trie persistence frontiers described by the input. + SaveBlocks(SaveBlocksInput, CrossbeamSender), /// Removes block data above the given block number from the database. /// @@ -311,20 +304,17 @@ impl PersistenceHandle { self.sender.send(action) } - /// Tells the persistence service to save a certain list of finalized blocks. The blocks are - /// assumed to be ordered by block number. + /// Tells the persistence service to advance its block-data and state/trie frontiers. /// /// This returns the latest hash that has been saved, allowing removal of that block and any /// previous blocks from in-memory data structures. This value is returned in the receiver end /// of the sender argument. - /// - /// If there are no blocks to persist, then `None` is sent in the sender. pub fn save_blocks( &self, - blocks: Vec>, + input: SaveBlocksInput, tx: CrossbeamSender, ) -> Result<(), SendError>> { - self.send_action(PersistenceAction::SaveBlocks(blocks, tx)) + self.send_action(PersistenceAction::SaveBlocks(input, tx)) } /// Queues the finalized block number to be persisted on disk. @@ -389,7 +379,8 @@ mod tests { use super::*; use alloy_eips::NumHash; use alloy_primitives::{BlockHash, BlockNumber, Bytes, B256, U256}; - use reth_chain_state::test_utils::TestBlockBuilder; + use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; + use reth_db_common::init::init_genesis; use reth_exex_types::FinishedExExHeight; use reth_provider::{ providers::{ProviderFactoryBuilder, ReadOnlyConfig}, @@ -404,6 +395,7 @@ mod tests { fn default_persistence_handle() -> PersistenceHandle { let provider = create_test_provider_factory(); + init_genesis(&provider).unwrap(); let (_finished_exex_height_tx, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs); @@ -415,6 +407,18 @@ mod tests { PersistenceHandle::::spawn_service(provider, pruner, sync_metrics_tx) } + fn full_save_input( + blocks: Vec>, + ) -> SaveBlocksInput { + let prev_tip = blocks + .first() + .map(|block| block.recovered_block().number.saturating_sub(1)) + .expect("save input must not be empty"); + let new_tip = + blocks.last().map(|block| block.recovered_block().number).expect("checked non-empty"); + SaveBlocksInput::new(blocks, prev_tip, prev_tip, new_tip, new_tip) + } + #[test] fn test_pruner_prunes_bal_store() { reth_tracing::init_test_tracing(); @@ -487,31 +491,17 @@ mod tests { } } - #[test] - fn test_save_blocks_empty() { - reth_tracing::init_test_tracing(); - let handle = default_persistence_handle(); - - let blocks = vec![]; - let (tx, rx) = crossbeam_channel::bounded(1); - - handle.save_blocks(blocks, tx).unwrap(); - - let result = rx.recv().unwrap(); - assert!(result.last_block.is_none()); - } - #[test] fn test_save_blocks_single_block() { reth_tracing::init_test_tracing(); let handle = default_persistence_handle(); - let block_number = 0; + let block_number = 1; let mut test_block_builder = TestBlockBuilder::eth(); let executed = test_block_builder.get_executed_block_with_number(block_number, B256::random()); let block_hash = executed.recovered_block().hash(); - let blocks = vec![executed]; + let blocks = full_save_input(vec![executed]); let (tx, rx) = crossbeam_channel::bounded(1); handle.save_blocks(blocks, tx).unwrap(); @@ -527,11 +517,11 @@ mod tests { let handle = default_persistence_handle(); let mut test_block_builder = TestBlockBuilder::eth(); - let blocks = test_block_builder.get_executed_blocks(0..5).collect::>(); + let blocks = test_block_builder.get_executed_blocks(1..6).collect::>(); let last_hash = blocks.last().unwrap().recovered_block().hash(); let (tx, rx) = crossbeam_channel::bounded(1); - handle.save_blocks(blocks, tx).unwrap(); + handle.save_blocks(full_save_input(blocks), tx).unwrap(); let result = rx.recv().unwrap(); assert_eq!(last_hash, result.last_block.unwrap().hash); } @@ -541,14 +531,14 @@ mod tests { reth_tracing::init_test_tracing(); let handle = default_persistence_handle(); - let ranges = [0..1, 1..2, 2..4, 4..5]; + let ranges = [1..2, 2..3, 3..5, 5..6]; let mut test_block_builder = TestBlockBuilder::eth(); for range in ranges { let blocks = test_block_builder.get_executed_blocks(range).collect::>(); let last_hash = blocks.last().unwrap().recovered_block().hash(); let (tx, rx) = crossbeam_channel::bounded(1); - handle.save_blocks(blocks, tx).unwrap(); + handle.save_blocks(full_save_input(blocks), tx).unwrap(); let result = rx.recv().unwrap(); assert_eq!(last_hash, result.last_block.unwrap().hash); @@ -623,29 +613,27 @@ mod tests { .expect("failed to open read-only provider factory"); secondary.set_storage_settings_cache(reth_provider::StorageSettings::v2()); - // --- Phase 1: Write blocks 0..3 via the primary --- + // --- Phase 1: Write blocks 1 and 2 via the primary --- + let genesis_hash = init_genesis(&provider_factory).unwrap(); let mut test_block_builder = TestBlockBuilder::eth().with_state(); let signer = test_block_builder.signer; - let blocks_a: Vec<_> = test_block_builder.get_executed_blocks(0..3).collect(); - let hash_a1 = blocks_a[1].recovered_block().hash(); - let hash_a2 = blocks_a[2].recovered_block().hash(); + let initial_balance = U256::from(10).pow(U256::from(18)); + let block_a1 = test_block_builder.get_executed_block_with_number(1, genesis_hash); + let hash_a1 = block_a1.recovered_block().hash(); + let block_a2 = test_block_builder.get_executed_block_with_number(2, hash_a1); + let hash_a2 = block_a2.recovered_block().hash(); - // Compute expected signer state after each block from tx counts. + // Compute expected signer state after block 1 from its transaction count. let single_cost = TestBlockBuilder::::single_tx_cost(); - let initial_balance = U256::from(10).pow(U256::from(18)); - let txs_in_block0 = blocks_a[0].recovered_block().body().transactions.len() as u64; - let txs_in_block1 = blocks_a[1].recovered_block().body().transactions.len() as u64; + let txs_in_block1 = block_a1.recovered_block().body().transactions.len() as u64; - let balance_after_block0 = initial_balance - single_cost * U256::from(txs_in_block0); - let nonce_after_block0 = txs_in_block0; - let balance_after_block1 = balance_after_block0 - single_cost * U256::from(txs_in_block1); - let nonce_after_block1 = nonce_after_block0 + txs_in_block1; + let balance_after_block1 = initial_balance - single_cost * U256::from(txs_in_block1); + let nonce_after_block1 = txs_in_block1; - { - let provider_rw = provider_factory.database_provider_rw().unwrap(); - provider_rw.save_blocks(blocks_a, SaveBlocksMode::Full).unwrap(); - provider_rw.commit().unwrap(); - } + let provider_rw = provider_factory.database_provider_rw().unwrap(); + let input = SaveBlocksInput::new(vec![block_a1, block_a2], 0, 0, 2, 2); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); // Secondary catches up and sees all 3 blocks. // Hold this provider (and its MDBX RO tx) across the reorg to test snapshot isolation. @@ -700,7 +688,8 @@ mod tests { provider_rw.commit().unwrap(); let provider_rw = pf.database_provider_rw().unwrap(); - provider_rw.save_blocks(vec![block_b2], SaveBlocksMode::Full).unwrap(); + let input = SaveBlocksInput::new(vec![block_b2], 1, 1, 2, 2); + provider_rw.save_blocks(&input).unwrap(); provider_rw.commit().unwrap(); }); diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 2507d45ee72..8fae1c7410a 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -30,9 +30,9 @@ use reth_primitives_traits::{ }; use reth_provider::{ BalProvider, BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader, - DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader, - StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, - StorageSettingsCache, TransactionVariant, + DatabaseProviderFactory, HashedPostStateProvider, ProviderError, SaveBlocksInput, + StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, + StorageChangeSetReader, StorageSettingsCache, TransactionVariant, }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; @@ -1427,10 +1427,19 @@ where .max_by_key(|block| block.recovered_block().number()) .map(|b| b.recovered_block().num_hash()) .expect("Checked non-empty persisting blocks"); - debug!(target: "engine::tree", count=blocks_to_persist.len(), blocks = ?blocks_to_persist.iter().map(|block| block.recovered_block().num_hash()).collect::>(), "Persisting blocks"); + + let previous_tip = self.persistence_state.last_persisted_block.number; + let input = SaveBlocksInput::new( + blocks_to_persist, + previous_tip, + previous_tip, + highest_num_hash.number, + highest_num_hash.number, + ); + let (tx, rx) = crossbeam_channel::bounded(1); - let _ = self.persistence.save_blocks(blocks_to_persist, tx); + let _ = self.persistence.save_blocks(input, tx); self.persistence_state.start_save(highest_num_hash, rx); } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index a6cd24e5cbd..af5580ddc7f 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -580,12 +580,14 @@ async fn test_tree_persist_blocks() { let received_action = test_harness.action_rx.recv().expect("Failed to receive save blocks action"); - if let PersistenceAction::SaveBlocks(saved_blocks, _) = received_action { + if let PersistenceAction::SaveBlocks(input, _) = received_action { // only blocks.len() - tree_config.memory_block_buffer_target() will be // persisted let expected_persist_len = blocks.len() - tree_config.memory_block_buffer_target() as usize; - assert_eq!(saved_blocks.len(), expected_persist_len); - assert_eq!(saved_blocks, blocks[..expected_persist_len]); + assert_eq!(input.persist_rest_blocks().len(), expected_persist_len); + assert_eq!(input.persist_rest_blocks(), &blocks[..expected_persist_len]); + assert_eq!(input.prev_db_tip(), input.prev_partial_state_trie()); + assert_eq!(input.new_db_tip(), input.new_partial_state_trie()); } else { panic!("unexpected action received {received_action:?}"); } @@ -1005,10 +1007,10 @@ async fn test_tree_state_on_new_head_reorg() { // get rid of the prev action let received_action = test_harness.action_rx.recv().unwrap(); - let PersistenceAction::SaveBlocks(saved_blocks, sender) = received_action else { + let PersistenceAction::SaveBlocks(input, sender) = received_action else { panic!("received wrong action"); }; - assert_eq!(saved_blocks, vec![blocks[0].clone(), blocks[1].clone()]); + assert_eq!(input.persist_rest_blocks(), &blocks[..2]); // send the response so we can advance again sender @@ -2286,15 +2288,14 @@ mod forkchoice_updated_tests { break; } - if let Ok(PersistenceAction::SaveBlocks(saved_blocks, sender)) = + if let Ok(PersistenceAction::SaveBlocks(input, sender)) = action_rx.recv_timeout(std::time::Duration::from_millis(100)) { - if let Some(last) = saved_blocks.last() { - last_persisted_number = last.recovered_block().number; - } + let last = input.last_block(); + last_persisted_number = last.number; sender .send(PersistenceResult { - last_block: saved_blocks.last().map(|b| b.recovered_block().num_hash()), + last_block: Some(last), commit_duration: Some(Duration::ZERO), }) .unwrap(); diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 056cb45856a..28c7c30058b 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -24,7 +24,7 @@ pub mod providers; pub use providers::{ DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, HistoricalStateProvider, HistoricalStateProviderRef, LatestStateProvider, LatestStateProviderRef, ProviderFactory, - PruneShardOutcome, PrunedIndices, SaveBlocksMode, StaticFileAccess, StaticFileProviderBuilder, + PruneShardOutcome, PrunedIndices, SaveBlocksInput, StaticFileAccess, StaticFileProviderBuilder, StaticFileWriteCtx, StaticFileWriter, }; diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 1b1b93bda8b..28729865bbc 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1015,7 +1015,7 @@ mod tests { create_test_provider_factory, create_test_provider_factory_with_chain_spec, MockNodeTypesWithDB, }, - BlockWriter, CanonChainTracker, ProviderFactory, SaveBlocksMode, + BlockWriter, CanonChainTracker, ProviderFactory, SaveBlocksInput, }; use alloy_consensus::constants::EMPTY_ROOT_HASH; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; @@ -1239,7 +1239,14 @@ mod tests { // Push to disk let provider_rw = hook_provider.database_provider_rw().unwrap(); - provider_rw.save_blocks(vec![lowest_memory_block], SaveBlocksMode::Full).unwrap(); + let input = SaveBlocksInput::new( + vec![lowest_memory_block], + state.anchor().number, + state.anchor().number, + block_number, + block_number, + ); + provider_rw.save_blocks(&input).unwrap(); provider_rw.commit().unwrap(); // Remove from memory diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 8686d51728e..7d6186c451c 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -47,9 +47,10 @@ use std::{ use tracing::{info, instrument, trace, warn}; mod provider; -pub use provider::{ - CommitOrder, DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, SaveBlocksMode, -}; +pub use provider::{CommitOrder, DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW}; + +mod save_blocks; +pub use save_blocks::SaveBlocksInput; use super::ProviderNodeTypes; use reth_trie::KeccakKeyHasher; diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 033f6190374..714f79cfd18 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -1,3 +1,4 @@ +use super::SaveBlocksInput; use crate::{ changesets_utils::StorageRevertsIter, providers::{ @@ -57,7 +58,7 @@ use reth_primitives_traits::{ use reth_prune_types::{ PruneCheckpoint, PruneMode, PruneModes, PruneSegment, MINIMUM_UNWIND_SAFE_DISTANCE, }; -use reth_stages_types::{StageCheckpoint, StageId}; +use reth_stages_types::{FinishCheckpoint, StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ BlockBodyIndicesProvider, BlockBodyReader, MetadataProvider, MetadataWriter, @@ -164,9 +165,9 @@ impl From> } } -/// Mode for [`DatabaseProvider::save_blocks`]. +/// Mode for [`DatabaseProvider::save_blocks_inner`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SaveBlocksMode { +enum SaveBlocksMode { /// Full mode: write block structure + receipts + state + trie. /// Used by engine/production code. Full, @@ -178,7 +179,7 @@ pub enum SaveBlocksMode { impl SaveBlocksMode { /// Returns `true` if this is [`SaveBlocksMode::Full`]. - pub const fn with_state(self) -> bool { + const fn with_state(self) -> bool { matches!(self, Self::Full) } } @@ -568,25 +569,51 @@ impl DatabaseProvider) -> ProviderResult<()> { + let (db_tip, partial_state_trie) = self + .get_stage_checkpoint(StageId::Finish)? + .map(|checkpoint| { + let partial_state_trie = checkpoint + .finish_stage_checkpoint() + .and_then(|finish| finish.partial_state_trie()) + .unwrap_or(checkpoint.block_number); + (checkpoint.block_number, partial_state_trie) + }) + .unwrap_or_default(); + + if db_tip != input.prev_db_tip() || partial_state_trie != input.prev_partial_state_trie() { + return Err(ProviderError::other(std::io::Error::other(format!( + "persistence frontiers do not match Finish checkpoint: expected database/state-trie tips #{}/{}, got #{}/{}", + input.prev_db_tip(), + input.prev_partial_state_trie(), + db_tip, + partial_state_trie, + )))) + } + + self.save_blocks_inner( + input.persist_rest_blocks(), + input.state_trie_blocks(), + input.state_trie_masking_blocks(), + (input.new_partial_state_trie() < input.new_db_tip()) + .then_some(input.new_partial_state_trie()), + SaveBlocksMode::Full, + ) + } + + fn save_blocks_inner( &self, - blocks: Vec>, + blocks: &[ExecutedBlock], + state_trie_blocks: &[ExecutedBlock], + state_trie_masking_blocks: &[ExecutedBlock], + partial_state_trie: Option, save_mode: SaveBlocksMode, ) -> ProviderResult<()> { - if blocks.is_empty() { - debug!(target: "providers::db", "Attempted to write empty block range"); - return Ok(()) - } - let total_start = Instant::now(); let block_count = blocks.len() as u64; let first_number = blocks.first().unwrap().recovered_block().number(); @@ -605,7 +632,7 @@ impl DatabaseProvider = { let mut nums = SmallVec::with_capacity(blocks.len()); let mut current = first_tx_num; - for block in &blocks { + for block in blocks { nums.push(current); current += block.recovered_block().body().transaction_count() as u64; } @@ -637,7 +664,7 @@ impl DatabaseProvider DatabaseProvider DatabaseProvider>(); + let mask = state_trie_masking_blocks + .iter() + .map(|block| block.trie_data.get().sorted.hashed_state.as_ref()) + .collect::>(); + let merged_hashed_state = + HashedPostStateSorted::disjointed_merge_batch(&batch, &mask); if !merged_hashed_state.is_empty() { self.write_hashed_state(&merged_hashed_state)?; } timings.write_hashed_state += start.elapsed(); let start = Instant::now(); - let merged_trie = - TrieUpdatesSorted::merge_batch(blocks.iter().rev().map(|b| b.trie_updates())); + let batch = state_trie_blocks + .iter() + .map(|block| block.trie_data.get().sorted.trie_updates.as_ref()) + .collect::>(); + let mask = state_trie_masking_blocks + .iter() + .map(|block| block.trie_data.get().sorted.trie_updates.as_ref()) + .collect::>(); + let merged_trie = TrieUpdatesSorted::disjointed_merge_batch(&batch, &mask); if !merged_trie.is_empty() { self.write_trie_updates_sorted(&merged_trie)?; } @@ -754,6 +794,16 @@ impl DatabaseProvider StageCheckpoint::new(last_block_number) + .with_finish_stage_checkpoint(FinishCheckpoint { + partial_state_trie: Some(partial_state_trie), + }), + None => StageCheckpoint::new(last_block_number), + }; + self.save_stage_checkpoint(StageId::Finish, checkpoint)?; + } timings.update_pipeline_stages = start.elapsed(); timings.mdbx = mdbx_start.elapsed(); @@ -3536,8 +3586,13 @@ impl BlockWriter ComputedTrieData::default(), ); - // Delegate to save_blocks with BlocksOnly mode (skips receipts/state/trie) - self.save_blocks(vec![executed_block], SaveBlocksMode::BlocksOnly)?; + self.save_blocks_inner( + std::slice::from_ref(&executed_block), + &[], + &[], + None, + SaveBlocksMode::BlocksOnly, + )?; // Return the body indices self.block_body_indices(block_number)? @@ -3959,6 +4014,8 @@ mod tests { map::{AddressMap, B256Map}, U256, }; + #[cfg(feature = "partial-persistence")] + use reth_chain_state::test_utils::TestBlockBuilder; use reth_chain_state::ExecutedBlock; use reth_db_api::models::StorageSettings; use reth_ethereum_primitives::Receipt; @@ -3973,6 +4030,26 @@ mod tests { use revm::{database::BundleState, state::AccountInfo}; use std::{sync::mpsc, time::Duration}; + /// Seeds block zero through the writer core because [`SaveBlocksInput`] only describes + /// advancing an existing persistence frontier. + fn save_genesis( + provider: &DatabaseProvider, + genesis: &ExecutedBlock, + ) -> ProviderResult<()> + where + TX: DbTx + DbTxMut + 'static, + N: NodeTypesForProvider, + { + assert_eq!(genesis.recovered_block().number(), 0); + provider.save_blocks_inner( + std::slice::from_ref(genesis), + std::slice::from_ref(genesis), + &[], + None, + SaveBlocksMode::Full, + ) + } + #[test] fn test_receipts_by_block_range_empty_range() { let factory = create_test_provider_factory(); @@ -4474,6 +4551,224 @@ mod tests { provider_rw.commit().unwrap(); } + #[cfg(feature = "partial-persistence")] + #[test] + fn test_save_blocks_only_masks_trie_with_deferred_blocks() { + use reth_trie::{ + updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, + BranchNodeCompact, HashedPostStateSorted, HashedStorageSorted, + }; + + fn branch(mask: u16) -> BranchNodeCompact { + BranchNodeCompact::new(mask, 0, 0, vec![], None) + } + + let factory = create_test_provider_factory(); + factory.set_storage_settings_cache(StorageSettings::v1()); + + let mut test_block_builder = TestBlockBuilder::eth().with_state(); + let genesis = test_block_builder.get_executed_blocks(0..1).next().unwrap(); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..3).collect(); + + let provider_rw = factory.provider_rw().unwrap(); + save_genesis(&provider_rw, &genesis).unwrap(); + provider_rw.commit().unwrap(); + + let kept_account = B256::with_last_byte(0x11); + let masked_account = B256::with_last_byte(0x12); + let kept_storage = B256::with_last_byte(0x21); + let masked_storage = B256::with_last_byte(0x22); + let kept_slot = B256::with_last_byte(0x31); + let masked_slot = B256::with_last_byte(0x32); + let kept_account_node = Nibbles::from_nibbles([0x1, 0x2]); + let masked_account_node = Nibbles::from_nibbles([0x1, 0x3]); + let kept_storage_node = Nibbles::from_nibbles([0x2, 0x1]); + let masked_storage_node = Nibbles::from_nibbles([0x2, 0x2]); + let full_persist_base = &blocks[0]; + let deferred_trie_base = &blocks[1]; + + let full_persist_hashed_state = HashedPostStateSorted::new( + vec![ + (kept_account, Some(Account::default())), + (masked_account, Some(Account { nonce: 1, ..Default::default() })), + ], + B256Map::from_iter([ + ( + kept_storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(kept_slot, U256::from(1))], + }, + ), + ( + masked_storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(masked_slot, U256::from(2))], + }, + ), + ]), + ); + let full_persist_trie_updates = TrieUpdatesSorted::new( + vec![ + (kept_account_node, Some(branch(0b0000_1111_0000_1111))), + (masked_account_node, Some(branch(0b1111_0000_1111_0000))), + ], + B256Map::from_iter([ + ( + kept_storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(kept_storage_node, Some(branch(0b1010)))], + }, + ), + ( + masked_storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(masked_storage_node, Some(branch(0b0101)))], + }, + ), + ]), + ); + + let full_persist_block = ExecutedBlock::new( + Arc::clone(&full_persist_base.recovered_block), + Arc::clone(&full_persist_base.execution_output), + ComputedTrieData::new( + Arc::new(full_persist_hashed_state), + Arc::new(full_persist_trie_updates), + ), + ); + + let deferred_trie_hashed_state = HashedPostStateSorted::new( + vec![(masked_account, Some(Account { nonce: 3, ..Default::default() }))], + B256Map::from_iter([( + masked_storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(masked_slot, U256::from(4))], + }, + )]), + ); + let deferred_trie_updates = TrieUpdatesSorted::new( + vec![(masked_account_node, Some(branch(0b0011_0011)))], + B256Map::from_iter([( + masked_storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(masked_storage_node, Some(branch(0b1100)))], + }, + )]), + ); + let deferred_trie_block = ExecutedBlock::new( + Arc::clone(&deferred_trie_base.recovered_block), + Arc::clone(&deferred_trie_base.execution_output), + ComputedTrieData::new( + Arc::new(deferred_trie_hashed_state), + Arc::new(deferred_trie_updates), + ), + ); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new(vec![full_persist_block, deferred_trie_block], 0, 0, 2, 1); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let tx = provider.tx_ref(); + let finish_checkpoint = provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); + assert_eq!(finish_checkpoint.block_number, 2); + assert_eq!( + finish_checkpoint.finish_stage_checkpoint().unwrap().partial_state_trie, + Some(1) + ); + assert!(provider.block_hash(2).unwrap().is_some()); + + let mut hashed_accounts = tx.cursor_read::().unwrap(); + assert!(hashed_accounts.seek_exact(kept_account).unwrap().is_some()); + assert!(hashed_accounts.seek_exact(masked_account).unwrap().is_none()); + + let mut hashed_storages = tx.cursor_dup_read::().unwrap(); + assert!(hashed_storages.seek_by_key_subkey(kept_storage, kept_slot).unwrap().is_some()); + assert!(hashed_storages + .walk_dup(Some(masked_storage), None) + .unwrap() + .next() + .transpose() + .unwrap() + .is_none()); + + let mut account_trie = tx.cursor_read::().unwrap(); + assert!(account_trie.seek_exact(StoredNibbles(kept_account_node)).unwrap().is_some()); + assert!(account_trie.seek_exact(StoredNibbles(masked_account_node)).unwrap().is_none()); + + let mut storage_trie = tx.cursor_dup_read::().unwrap(); + let kept_entries: Vec<_> = storage_trie + .walk_dup(Some(kept_storage), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(kept_entries.len(), 1); + assert_eq!(kept_entries[0].1.nibbles.0, kept_storage_node); + + let masked_entries: Vec<_> = storage_trie + .walk_dup(Some(masked_storage), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(masked_entries.is_empty()); + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn test_save_blocks_partial_cycles_do_not_duplicate_static_file_writes() { + let factory = create_test_provider_factory(); + let mut test_block_builder = TestBlockBuilder::eth().with_state(); + + let genesis = test_block_builder.get_executed_blocks(0..1).next().unwrap(); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..5).collect(); + + let provider_rw = factory.provider_rw().unwrap(); + save_genesis(&provider_rw, &genesis).unwrap(); + provider_rw.commit().unwrap(); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new(blocks[..2].to_vec(), 0, 0, 2, 2); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new(blocks[2..].to_vec(), 2, 2, 4, 2); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let provider_rw = factory.provider_rw().unwrap(); + let stale_input = SaveBlocksInput::new(vec![blocks[0].clone()], 0, 0, 1, 1); + let err = provider_rw.save_blocks(&stale_input).unwrap_err(); + assert!(err.to_string().contains("persistence frontiers do not match Finish checkpoint")); + drop(provider_rw); + + let provider = factory.provider().unwrap(); + let finish_checkpoint = provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); + assert_eq!(finish_checkpoint.block_number, 4); + assert_eq!( + finish_checkpoint.finish_stage_checkpoint().unwrap().partial_state_trie, + Some(2) + ); + + let static_files = factory.static_file_provider(); + assert_eq!(static_files.get_highest_static_file_block(StaticFileSegment::Headers), Some(4)); + assert_eq!( + static_files.get_highest_static_file_block(StaticFileSegment::Transactions), + Some(4) + ); + assert_eq!( + static_files.get_highest_static_file_block(StaticFileSegment::Receipts), + Some(4) + ); + } + #[test] fn test_prunable_receipts_logic() { let insert_blocks = @@ -5045,7 +5340,7 @@ mod tests { Default::default(), ); - let genesis_executed = ExecutedBlock::new( + let genesis_executed: ExecutedBlock = ExecutedBlock::new( Arc::new(genesis.try_recover().unwrap()), Arc::new(BlockExecutionOutput { result: BlockExecutionResult { @@ -5059,7 +5354,7 @@ mod tests { ComputedTrieData::default(), ); let provider_rw = factory.provider_rw().unwrap(); - provider_rw.save_blocks(vec![genesis_executed], SaveBlocksMode::Full).unwrap(); + save_genesis(&provider_rw, &genesis_executed).unwrap(); provider_rw.commit().unwrap(); let mut blocks: Vec = Vec::new(); @@ -5133,7 +5428,8 @@ mod tests { } let provider_rw = factory.provider_rw().unwrap(); - provider_rw.save_blocks(blocks, SaveBlocksMode::Full).unwrap(); + let input = SaveBlocksInput::new(blocks, 0, 0, num_blocks, num_blocks); + provider_rw.save_blocks(&input).unwrap(); provider_rw.commit().unwrap(); let provider = factory.provider().unwrap(); diff --git a/crates/storage/provider/src/providers/database/save_blocks.rs b/crates/storage/provider/src/providers/database/save_blocks.rs new file mode 100644 index 00000000000..66674f3ff54 --- /dev/null +++ b/crates/storage/provider/src/providers/database/save_blocks.rs @@ -0,0 +1,213 @@ +use alloy_consensus::BlockHeader; +use alloy_eips::BlockNumHash; +use alloy_primitives::BlockNumber; +use reth_chain_state::ExecutedBlock; +use reth_ethereum_primitives::EthPrimitives; +use reth_primitives_traits::NodePrimitives; + +/// Input for advancing the engine's two persistence frontiers. +/// +/// The Finish checkpoint's block number (`db_tip`) tracks complete block, execution, and history +/// data. `Finish.partial_state_trie` tracks the hashed-state/trie frontier, which may lag behind it +/// while a suffix of blocks is retained in memory as a mask. The hashed-state/trie tables are not +/// a complete snapshot at `db_tip` while this suffix exists; the database and in-memory mask +/// together represent the canonical state. +/// +/// `blocks` contains every canonical block in `(prev_partial_state_trie, new_db_tip]`, ordered from +/// oldest to newest. The four frontiers derive the ranges written during this operation: +/// +/// - `(prev_db_tip, new_db_tip]`: persist block, execution, and history data; +/// - `(prev_partial_state_trie, new_partial_state_trie]`: consider hashed-state/trie updates for +/// persistence; and +/// - `(new_partial_state_trie, new_db_tip]`: retain as the masking suffix and do not persist its +/// hashed-state/trie updates. +/// +/// Updates in the second range that are overwritten by the masking suffix are also omitted. This +/// is the work partial persistence avoids: only the newest value needs to be applied later when +/// the masking block leaves memory. If both new frontiers are equal, the masking suffix is empty +/// and hashed state/trie are fully flushed through `new_db_tip`. +/// +/// Genesis must already be initialized because this input only advances an existing database tip. +#[derive(Debug)] +pub struct SaveBlocksInput { + blocks: Vec>, + prev_db_tip: BlockNumber, + prev_partial_state_trie: BlockNumber, + new_db_tip: BlockNumber, + new_partial_state_trie: BlockNumber, +} + +impl SaveBlocksInput { + /// Creates an input that advances the existing frontiers to `new_db_tip` and + /// `new_partial_state_trie`. + /// + /// `new_partial_state_trie` may remain behind `prev_db_tip`. This occurs while a masking + /// window is first growing: new block data advances without forcing all previously masked + /// hashed-state/trie updates to disk. + /// + /// # Panics + /// + /// Panics if the database frontier does not advance, the state/trie frontier moves backwards + /// or exceeds the database frontier, or `blocks` is not the exact contiguous range + /// `(prev_partial_state_trie, new_db_tip]`. + pub fn new( + blocks: Vec>, + prev_db_tip: BlockNumber, + prev_partial_state_trie: BlockNumber, + new_db_tip: BlockNumber, + new_partial_state_trie: BlockNumber, + ) -> Self { + assert!( + prev_partial_state_trie <= prev_db_tip, + "previous state/trie tip must not exceed previous database tip" + ); + assert!(prev_db_tip < new_db_tip, "database tip must advance"); + assert!( + prev_partial_state_trie <= new_partial_state_trie, + "state/trie tip must not move backwards" + ); + assert!( + new_partial_state_trie <= new_db_tip, + "new state/trie tip must not exceed new database tip" + ); + + let expected_len = new_db_tip.saturating_sub(prev_partial_state_trie) as usize; + assert_eq!( + blocks.len(), + expected_len, + "blocks must cover (prev_partial_state_trie, new_db_tip]" + ); + assert!( + blocks.iter().enumerate().all(|(index, block)| { + block.recovered_block().number() == prev_partial_state_trie + index as u64 + 1 + }), + "blocks must cover (prev_partial_state_trie, new_db_tip]" + ); + + Self { blocks, prev_db_tip, prev_partial_state_trie, new_db_tip, new_partial_state_trie } + } + + /// Returns the previous Finish checkpoint block number. + pub const fn prev_db_tip(&self) -> BlockNumber { + self.prev_db_tip + } + + /// Returns the previous `Finish.partial_state_trie` frontier. + pub const fn prev_partial_state_trie(&self) -> BlockNumber { + self.prev_partial_state_trie + } + + /// Returns the new Finish checkpoint block number. + pub const fn new_db_tip(&self) -> BlockNumber { + self.new_db_tip + } + + /// Returns the new `Finish.partial_state_trie` frontier. + pub const fn new_partial_state_trie(&self) -> BlockNumber { + self.new_partial_state_trie + } + + /// Returns the block at the new Finish checkpoint. + pub fn last_block(&self) -> BlockNumHash { + self.blocks + .last() + .expect("constructor ensures a non-empty block range") + .recovered_block() + .num_hash() + } + + /// Returns the first newly persisted block. + pub fn first_persist_rest_block(&self) -> &ExecutedBlock { + self.persist_rest_blocks() + .first() + .expect("constructor ensures a non-empty persistence range") + } + + /// Returns newly persisted blocks whose block, execution, and history data should be written. + pub fn persist_rest_blocks(&self) -> &[ExecutedBlock] { + &self.blocks[(self.prev_db_tip - self.prev_partial_state_trie) as usize..] + } + + /// Returns all blocks whose hashed-state/trie updates are candidates for persistence. + /// + /// Updates overwritten by [`Self::state_trie_masking_blocks`] are filtered out by the writer. + pub fn state_trie_blocks(&self) -> &[ExecutedBlock] { + &self.blocks[..(self.new_partial_state_trie - self.prev_partial_state_trie) as usize] + } + + /// Returns the fixed suffix whose hashed-state/trie updates remain in memory. + /// + /// This suffix suppresses older updates to the same hashed keys and trie nodes. It is also the + /// overlay required to use the database at the new Finish checkpoint. + pub fn state_trie_masking_blocks(&self) -> &[ExecutedBlock] { + &self.blocks[(self.new_partial_state_trie - self.prev_partial_state_trie) as usize..] + } +} + +#[cfg(test)] +mod tests { + use super::*; + use reth_chain_state::test_utils::TestBlockBuilder; + + #[test] + fn splits_frontier_ranges() { + let blocks = TestBlockBuilder::eth().get_executed_blocks(6..23).collect(); + let input = SaveBlocksInput::new(blocks, 11, 5, 22, 16); + + assert_eq!( + input + .persist_rest_blocks() + .iter() + .map(|block| block.recovered_block().number()) + .collect::>(), + (12..=22).collect::>() + ); + assert_eq!( + input + .state_trie_blocks() + .iter() + .map(|block| block.recovered_block().number()) + .collect::>(), + (6..=16).collect::>() + ); + assert_eq!( + input + .state_trie_masking_blocks() + .iter() + .map(|block| block.recovered_block().number()) + .collect::>(), + (17..=22).collect::>() + ); + } + + #[test] + fn grows_masking_window_without_forcing_new_state_writes() { + let blocks = TestBlockBuilder::eth().get_executed_blocks(13..17).collect(); + let input = SaveBlocksInput::new(blocks, 15, 12, 16, 13); + + assert_eq!(input.first_persist_rest_block().recovered_block().number(), 16); + assert_eq!(input.state_trie_blocks()[0].recovered_block().number(), 13); + assert_eq!( + input + .state_trie_masking_blocks() + .iter() + .map(|block| block.recovered_block().number()) + .collect::>(), + vec![14, 15, 16] + ); + } + + #[test] + #[should_panic(expected = "database tip must advance")] + fn requires_database_tip_to_advance() { + let blocks = TestBlockBuilder::eth().get_executed_blocks(1..2).collect(); + let _ = SaveBlocksInput::new(blocks, 1, 0, 1, 1); + } + + #[test] + #[should_panic(expected = "blocks must cover (prev_partial_state_trie, new_db_tip]")] + fn requires_contiguous_blocks() { + let blocks = TestBlockBuilder::eth().get_executed_blocks(2..4).collect(); + let _ = SaveBlocksInput::new(blocks, 0, 0, 2, 2); + } +} diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index 454bf947e7b..89a7380504d 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -3,7 +3,7 @@ use core::ops::Not; use crate::{ added_removed_keys::MultiAddedRemovedKeys, prefix_set::{PrefixSetMut, TriePrefixSetsMut}, - utils::{extend_sorted_vec, kway_merge_sorted}, + utils::{extend_sorted_vec, kway_merge_disjoint_sorted, kway_merge_sorted}, KeyHasher, MultiProofTargets, Nibbles, }; use alloc::{borrow::Cow, vec::Vec}; @@ -694,6 +694,95 @@ impl HashedPostStateSorted { Self { accounts, storages } } + /// Merges the batch and removes overlapping keys whose mask values all differ from the merged + /// batch value. + /// + /// Account keys are masked at the top level, while storage entries are masked at the slot + /// level. For duplicate keys in the batch, later items take precedence over earlier ones. An + /// overlapping entry is retained if any mask value is equal to the merged batch value. The + /// order of the mask does not matter. An empty mask merges the batch without filtering. + /// + /// # Panics + /// + /// Panics if any batch or mask entry wipes an entire storage. + pub fn disjointed_merge_batch<'a>(batch: &[&'a Self], mask: &[&'a Self]) -> Self { + let account_count = batch.iter().map(|item| item.accounts.len()).sum(); + let mut accounts = Vec::with_capacity(account_count); + accounts.extend(kway_merge_disjoint_sorted( + batch.iter().rev().map(|item| item.accounts.as_slice()), + mask.iter().map(|item| item.accounts.as_slice()), + )); + + struct StorageAcc<'a> { + slot_count: usize, + slices: Vec<&'a [(B256, U256)]>, + } + + #[derive(Default)] + struct StorageMaskAcc<'a> { + slices: Vec<&'a [(B256, U256)]>, + } + + let mut storages = B256Map::with_capacity_and_hasher( + batch.iter().map(|item| item.storages.len()).sum(), + Default::default(), + ); + + for item in batch.iter().rev() { + for (hashed_address, storage) in &item.storages { + assert!( + !storage.wiped, + "storage wipes are not supported by disjointed_merge_batch" + ); + let entry = storages + .entry(*hashed_address) + .or_insert_with(|| StorageAcc { slot_count: 0, slices: Vec::new() }); + entry.slices.push(storage.storage_slots.as_slice()); + entry.slot_count += storage.storage_slots.len(); + } + } + + let mut storage_masks: B256Map> = B256Map::with_capacity_and_hasher( + mask.iter().map(|item| item.storages.len()).sum(), + Default::default(), + ); + for item in mask { + for (hashed_address, storage) in &item.storages { + assert!( + !storage.wiped, + "storage wipes are not supported by disjointed_merge_batch" + ); + let entry = storage_masks.entry(*hashed_address).or_default(); + entry.slices.push(storage.storage_slots.as_slice()); + } + } + + let storages = storages + .into_iter() + .filter_map(|(hashed_address, entry)| { + let slot_count = entry.slot_count; + let storage_slots = match storage_masks.get(&hashed_address) { + Some(mask_entry) => { + let mut storage_slots = Vec::with_capacity(slot_count); + storage_slots.extend(kway_merge_disjoint_sorted( + entry.slices, + mask_entry.slices.iter().copied(), + )); + storage_slots + } + None => kway_merge_sorted(entry.slices), + }; + + (!storage_slots.is_empty() || mask.is_empty()).then_some(( + hashed_address, + HashedStorageSorted { wiped: false, storage_slots }, + )) + }) + .collect(); + + Self { accounts, storages } + } + /// Clears all accounts and storage data. pub fn clear(&mut self) { self.accounts.clear(); @@ -1615,6 +1704,218 @@ mod tests { assert_eq!(state.accounts.get(&addr1), Some(&None)); } + #[test] + fn test_hashed_post_state_sorted_disjointed_merge_batch() { + fn account(nonce: u64) -> Account { + Account { nonce, balance: U256::ZERO, bytecode_hash: None } + } + + let kept_account = B256::with_last_byte(1); + let removed_account = B256::with_last_byte(2); + let kept_storage = B256::with_last_byte(3); + let slot1 = B256::with_last_byte(11); + let slot2 = B256::with_last_byte(12); + + let older = HashedPostStateSorted::new( + vec![(kept_account, Some(account(1))), (removed_account, Some(account(10)))], + B256Map::from_iter([( + kept_storage, + HashedStorageSorted { wiped: false, storage_slots: vec![(slot1, U256::from(1))] }, + )]), + ); + + let newer = HashedPostStateSorted::new( + vec![(kept_account, Some(account(2)))], + B256Map::from_iter([( + kept_storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(slot1, U256::from(3)), (slot2, U256::from(4))], + }, + )]), + ); + + let remove_a = HashedPostStateSorted::new( + vec![(removed_account, None)], + B256Map::from_iter([( + kept_storage, + HashedStorageSorted { wiped: false, storage_slots: vec![(slot2, U256::ZERO)] }, + )]), + ); + + let remove_b = HashedPostStateSorted::new( + vec![(B256::with_last_byte(255), Some(account(99)))], + B256Map::default(), + ); + + let result = HashedPostStateSorted::disjointed_merge_batch( + &[&older, &newer], + &[&remove_b, &remove_a], + ); + + assert_eq!(result.accounts, vec![(kept_account, Some(account(2)))]); + assert_eq!(result.storages.len(), 1); + assert_eq!( + result.storages.get(&kept_storage), + Some(&HashedStorageSorted { + wiped: false, + storage_slots: vec![(slot1, U256::from(3))], + }) + ); + } + + #[test] + fn test_hashed_post_state_sorted_disjointed_merge_batch_empty_mask_merges_batch() { + let address = B256::with_last_byte(1); + let storage = B256::with_last_byte(2); + let slot = B256::with_last_byte(3); + let empty_storage = B256::with_last_byte(4); + let older = HashedPostStateSorted::new( + vec![(address, Some(Account { nonce: 1, ..Default::default() }))], + B256Map::from_iter([ + ( + storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(slot, U256::from(1))], + }, + ), + (empty_storage, HashedStorageSorted::default()), + ]), + ); + let newer = HashedPostStateSorted::new( + vec![(address, Some(Account { nonce: 2, ..Default::default() }))], + B256Map::from_iter([( + storage, + HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(2))] }, + )]), + ); + let expected = HashedPostStateSorted::merge_batch(vec![newer.clone(), older.clone()]); + + let result = HashedPostStateSorted::disjointed_merge_batch(&[&older, &newer], &[]); + + assert_eq!(result, expected); + } + + #[test] + fn test_hashed_post_state_sorted_disjointed_merge_batch_removes_overlapping_batch_key() { + fn account(nonce: u64) -> Account { + Account { nonce, balance: U256::ZERO, bytecode_hash: None } + } + + let overlapping_account = B256::with_last_byte(21); + + let older = HashedPostStateSorted::new( + vec![(overlapping_account, Some(account(1)))], + B256Map::default(), + ); + + let newer = HashedPostStateSorted::new( + vec![(overlapping_account, Some(account(2)))], + B256Map::default(), + ); + + let remove = + HashedPostStateSorted::new(vec![(overlapping_account, None)], B256Map::default()); + + let result = HashedPostStateSorted::disjointed_merge_batch(&[&older, &newer], &[&remove]); + + assert!(result.accounts.is_empty()); + } + + #[test] + fn test_hashed_post_state_sorted_disjointed_merge_batch_keeps_equal_overlaps() { + fn account(nonce: u64) -> Account { + Account { nonce, balance: U256::ZERO, bytecode_hash: None } + } + + let address = B256::with_last_byte(21); + let deleted_address = B256::with_last_byte(24); + let storage = B256::with_last_byte(22); + let deleted_storage = B256::with_last_byte(25); + let slot = B256::with_last_byte(23); + let deleted_slot = B256::with_last_byte(26); + let batch = HashedPostStateSorted::new( + vec![(address, Some(account(1))), (deleted_address, None)], + B256Map::from_iter([ + ( + storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(slot, U256::from(1))], + }, + ), + ( + deleted_storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(deleted_slot, U256::ZERO)], + }, + ), + ]), + ); + let different_mask = HashedPostStateSorted::new( + vec![(address, Some(account(2))), (deleted_address, Some(account(3)))], + B256Map::from_iter([ + ( + storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(slot, U256::from(2))], + }, + ), + ( + deleted_storage, + HashedStorageSorted { + wiped: false, + storage_slots: vec![(deleted_slot, U256::from(3))], + }, + ), + ]), + ); + let equal_mask = batch.clone(); + + let result = HashedPostStateSorted::disjointed_merge_batch( + &[&batch], + &[&different_mask, &equal_mask], + ); + let reversed = HashedPostStateSorted::disjointed_merge_batch( + &[&batch], + &[&equal_mask, &different_mask], + ); + + assert_eq!(result, batch); + assert_eq!(reversed, result); + } + + #[test] + fn test_hashed_post_state_sorted_disjointed_merge_batch_ignores_empty_storage_mask() { + let storage = B256::with_last_byte(31); + let slot = B256::with_last_byte(32); + + let batch = HashedPostStateSorted::new( + vec![], + B256Map::from_iter([( + storage, + HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(1))] }, + )]), + ); + let mask = HashedPostStateSorted::new( + vec![], + B256Map::from_iter([( + storage, + HashedStorageSorted { wiped: false, storage_slots: vec![] }, + )]), + ); + + let result = HashedPostStateSorted::disjointed_merge_batch(&[&batch], &[&mask]); + + assert_eq!( + result.storages.get(&storage), + Some(&HashedStorageSorted { wiped: false, storage_slots: vec![(slot, U256::from(1))] }) + ); + } + /// Test non-wiped storage merges both zero and non-zero valued slots #[test] fn test_hashed_storage_extend_from_sorted_non_wiped() { diff --git a/crates/trie/common/src/updates.rs b/crates/trie/common/src/updates.rs index d73b2c4d460..d5fbb7db395 100644 --- a/crates/trie/common/src/updates.rs +++ b/crates/trie/common/src/updates.rs @@ -1,5 +1,5 @@ use crate::{ - utils::{extend_sorted_vec, kway_merge_sorted}, + utils::{extend_sorted_vec, kway_merge_disjoint_sorted, kway_merge_sorted}, BranchNodeCompact, HashBuilder, Nibbles, }; use alloc::{ @@ -710,6 +710,95 @@ impl TrieUpdatesSorted { Self { account_nodes, storage_tries } } + + /// Merges the batch and removes overlapping keys whose mask values all differ from the merged + /// batch value. + /// + /// Account trie nodes are masked at the top level, while storage trie entries are masked at the + /// node level. For duplicate keys in the batch, later items take precedence over earlier ones. + /// An overlapping entry is retained if any mask value is equal to the merged batch value. The + /// order of the mask does not matter. An empty mask merges the batch without filtering. + /// + /// # Panics + /// + /// Panics if any batch or mask entry deletes an entire storage trie. + pub fn disjointed_merge_batch<'a>(batch: &[&'a Self], mask: &[&'a Self]) -> Self { + let account_node_count = batch.iter().map(|item| item.account_nodes.len()).sum(); + let mut account_nodes = Vec::with_capacity(account_node_count); + account_nodes.extend(kway_merge_disjoint_sorted( + batch.iter().rev().map(|item| item.account_nodes.as_slice()), + mask.iter().map(|item| item.account_nodes.as_slice()), + )); + + struct StorageAcc<'a> { + node_count: usize, + slices: Vec<&'a [(Nibbles, Option)]>, + } + + #[derive(Default)] + struct StorageMaskAcc<'a> { + slices: Vec<&'a [(Nibbles, Option)]>, + } + + let mut storage_tries = B256Map::with_capacity_and_hasher( + batch.iter().map(|item| item.storage_tries.len()).sum(), + Default::default(), + ); + + for item in batch.iter().rev() { + for (hashed_address, storage_trie) in &item.storage_tries { + assert!( + !storage_trie.is_deleted, + "storage wipes are not supported by disjointed_merge_batch" + ); + let entry = storage_tries + .entry(*hashed_address) + .or_insert_with(|| StorageAcc { node_count: 0, slices: Vec::new() }); + entry.slices.push(storage_trie.storage_nodes.as_slice()); + entry.node_count += storage_trie.storage_nodes.len(); + } + } + + let mut storage_masks: B256Map> = B256Map::with_capacity_and_hasher( + mask.iter().map(|item| item.storage_tries.len()).sum(), + Default::default(), + ); + for item in mask { + for (hashed_address, storage_trie) in &item.storage_tries { + assert!( + !storage_trie.is_deleted, + "storage wipes are not supported by disjointed_merge_batch" + ); + let entry = storage_masks.entry(*hashed_address).or_default(); + entry.slices.push(storage_trie.storage_nodes.as_slice()); + } + } + + let storage_tries = storage_tries + .into_iter() + .filter_map(|(hashed_address, entry)| { + let node_count = entry.node_count; + let storage_nodes = match storage_masks.get(&hashed_address) { + Some(mask_entry) => { + let mut storage_nodes = Vec::with_capacity(node_count); + storage_nodes.extend(kway_merge_disjoint_sorted( + entry.slices, + mask_entry.slices.iter().copied(), + )); + storage_nodes + } + None => kway_merge_sorted(entry.slices), + }; + + (!storage_nodes.is_empty() || mask.is_empty()).then_some(( + hashed_address, + StorageTrieUpdatesSorted { is_deleted: false, storage_nodes }, + )) + }) + .collect(); + + Self::new(account_nodes, storage_tries) + } } impl AsRef for TrieUpdatesSorted { @@ -977,6 +1066,273 @@ mod tests { assert_eq!(storage3.storage_nodes[1].0, Nibbles::from_nibbles_unchecked([0x07])); } + #[test] + fn test_trie_updates_sorted_disjointed_merge_batch() { + let kept_node = Nibbles::from_nibbles_unchecked([0x01]); + let removed_node = Nibbles::from_nibbles_unchecked([0x02]); + let kept_storage = B256::from([3; 32]); + let slot1 = Nibbles::from_nibbles_unchecked([0x0a]); + let slot2 = Nibbles::from_nibbles_unchecked([0x0b]); + + let older = TrieUpdatesSorted::new( + vec![(kept_node, Some(BranchNodeCompact::default())), (removed_node, None)], + B256Map::from_iter([( + kept_storage, + StorageTrieUpdatesSorted { is_deleted: false, storage_nodes: vec![(slot1, None)] }, + )]), + ); + + let newer = TrieUpdatesSorted::new( + vec![(kept_node, None)], + B256Map::from_iter([( + kept_storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(slot1, Some(BranchNodeCompact::default())), (slot2, None)], + }, + )]), + ); + + let remove_a = TrieUpdatesSorted::new( + vec![(removed_node, Some(BranchNodeCompact::default()))], + B256Map::from_iter([( + kept_storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(slot2, Some(BranchNodeCompact::default()))], + }, + )]), + ); + + let remove_b = TrieUpdatesSorted::new( + vec![(Nibbles::from_nibbles_unchecked([0x0f]), Some(BranchNodeCompact::default()))], + B256Map::default(), + ); + + let result = + TrieUpdatesSorted::disjointed_merge_batch(&[&older, &newer], &[&remove_b, &remove_a]); + + assert_eq!(result.account_nodes, vec![(kept_node, None)]); + assert_eq!(result.storage_tries.len(), 1); + assert_eq!( + result.storage_tries.get(&kept_storage), + Some(&StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(slot1, Some(BranchNodeCompact::default()))], + }) + ); + } + + #[test] + fn test_trie_updates_sorted_disjointed_merge_batch_empty_mask_merges_batch() { + let node = Nibbles::from_nibbles_unchecked([0x01]); + let storage = B256::with_last_byte(2); + let storage_node = Nibbles::from_nibbles_unchecked([0x03]); + let empty_storage = B256::with_last_byte(4); + let older = TrieUpdatesSorted::new( + vec![(node, Some(BranchNodeCompact::default()))], + B256Map::from_iter([ + ( + storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(storage_node, None)], + }, + ), + (empty_storage, StorageTrieUpdatesSorted::default()), + ]), + ); + let newer = TrieUpdatesSorted::new( + vec![(node, None)], + B256Map::from_iter([( + storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(storage_node, Some(BranchNodeCompact::default()))], + }, + )]), + ); + let expected = TrieUpdatesSorted::merge_batch(vec![newer.clone(), older.clone()]); + + let result = TrieUpdatesSorted::disjointed_merge_batch(&[&older, &newer], &[]); + + assert_eq!(result, expected); + } + + #[test] + fn test_trie_updates_sorted_disjointed_merge_batch_removes_overlapping_batch_key() { + let overlapping_node = Nibbles::from_nibbles_unchecked([0x03]); + + let older = TrieUpdatesSorted::new( + vec![(overlapping_node, Some(BranchNodeCompact::default()))], + B256Map::default(), + ); + + let newer = TrieUpdatesSorted::new(vec![(overlapping_node, None)], B256Map::default()); + + let remove = TrieUpdatesSorted::new( + vec![(overlapping_node, Some(BranchNodeCompact::default()))], + B256Map::default(), + ); + + let result = TrieUpdatesSorted::disjointed_merge_batch(&[&older, &newer], &[&remove]); + + assert!(result.account_nodes.is_empty()); + } + + #[test] + fn test_trie_updates_sorted_disjointed_merge_batch_keeps_equal_overlaps() { + fn branch(mask: u16) -> BranchNodeCompact { + BranchNodeCompact::new(mask, 0, 0, vec![], None) + } + + let node_path = Nibbles::from_nibbles_unchecked([0x03]); + let deleted_node_path = Nibbles::from_nibbles_unchecked([0x04]); + let storage = B256::from([5; 32]); + let deleted_storage = B256::from([6; 32]); + let storage_node_path = Nibbles::from_nibbles_unchecked([0x0c]); + let deleted_storage_node_path = Nibbles::from_nibbles_unchecked([0x0d]); + let batch = TrieUpdatesSorted::new( + vec![(node_path, Some(branch(0b1010_0101))), (deleted_node_path, None)], + B256Map::from_iter([ + ( + storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(storage_node_path, Some(branch(0b0011_1100)))], + }, + ), + ( + deleted_storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(deleted_storage_node_path, None)], + }, + ), + ]), + ); + let different_mask = TrieUpdatesSorted::new( + vec![ + (node_path, Some(branch(0b0101_1010))), + (deleted_node_path, Some(branch(0b1111_0000))), + ], + B256Map::from_iter([ + ( + storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(storage_node_path, Some(branch(0b1100_0011)))], + }, + ), + ( + deleted_storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(deleted_storage_node_path, Some(branch(0b0000_1111)))], + }, + ), + ]), + ); + let equal_mask = batch.clone(); + + let result = + TrieUpdatesSorted::disjointed_merge_batch(&[&batch], &[&different_mask, &equal_mask]); + let reversed = + TrieUpdatesSorted::disjointed_merge_batch(&[&batch], &[&equal_mask, &different_mask]); + + assert_eq!(result, batch); + assert_eq!(reversed, result); + } + + #[test] + fn test_trie_updates_sorted_disjointed_merge_batch_uses_exact_key_masking() { + let hashed_address = B256::from([7; 32]); + let grandparent = Nibbles::from_nibbles_unchecked([0x05]); + let parent = Nibbles::from_nibbles_unchecked([0x05, 0x04]); + let child = Nibbles::from_nibbles_unchecked([0x05, 0x04, 0x03]); + let different_node = BranchNodeCompact::new(1, 0, 0, vec![], None); + + let batch = TrieUpdatesSorted::new( + vec![ + (grandparent, Some(BranchNodeCompact::default())), + (parent, Some(BranchNodeCompact::default())), + (child, Some(BranchNodeCompact::default())), + ], + B256Map::from_iter([( + hashed_address, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![ + (grandparent, Some(BranchNodeCompact::default())), + (parent, Some(BranchNodeCompact::default())), + (child, Some(BranchNodeCompact::default())), + ], + }, + )]), + ); + let mask = TrieUpdatesSorted::new( + vec![ + (grandparent, Some(different_node.clone())), + (parent, Some(different_node.clone())), + ], + B256Map::from_iter([( + hashed_address, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![ + (grandparent, Some(different_node.clone())), + (parent, Some(different_node)), + ], + }, + )]), + ); + + let result = TrieUpdatesSorted::disjointed_merge_batch(&[&batch], &[&mask]); + + assert_eq!(result.account_nodes, vec![(child, Some(BranchNodeCompact::default()))]); + assert_eq!( + result.storage_tries.get(&hashed_address), + Some(&StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(child, Some(BranchNodeCompact::default()))], + }) + ); + } + + #[test] + fn test_trie_updates_sorted_disjointed_merge_batch_ignores_empty_storage_mask() { + let storage = B256::from([6; 32]); + let slot = Nibbles::from_nibbles_unchecked([0x0d]); + + let batch = TrieUpdatesSorted::new( + vec![], + B256Map::from_iter([( + storage, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(slot, Some(BranchNodeCompact::default()))], + }, + )]), + ); + let mask = TrieUpdatesSorted::new( + vec![], + B256Map::from_iter([( + storage, + StorageTrieUpdatesSorted { is_deleted: false, storage_nodes: vec![] }, + )]), + ); + + let result = TrieUpdatesSorted::disjointed_merge_batch(&[&batch], &[&mask]); + + assert_eq!( + result.storage_tries.get(&storage), + Some(&StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(slot, Some(BranchNodeCompact::default()))], + }) + ); + } + /// Test extending with storage tries adds both nodes and removed nodes correctly #[test] fn test_trie_updates_extend_from_sorted_with_storage_tries() { diff --git a/crates/trie/common/src/utils.rs b/crates/trie/common/src/utils.rs index 6d6f134a3ac..a01c7c38e1f 100644 --- a/crates/trie/common/src/utils.rs +++ b/crates/trie/common/src/utils.rs @@ -26,6 +26,64 @@ where .collect() } +/// Merge sorted left slices, excluding keys present in any right slice unless one of the right +/// values is equal to the selected left value. +/// Retained keys and values are cloned as the returned iterator is consumed. +/// +/// Callers pass left slices in priority order (index 0 = highest priority), so the first +/// left slice's value for a key takes precedence over later slices. Right slice order is ignored. +pub(crate) fn kway_merge_disjoint_sorted<'a, K, V>( + left_slices: impl IntoIterator, + right_slices: impl IntoIterator, +) -> impl Iterator +where + K: Ord + Clone + 'a, + V: Clone + PartialEq + 'a, +{ + let mut right_entries = right_slices + .into_iter() + .filter(|s| !s.is_empty()) + .map(|s| s.iter()) + .kmerge_by(|(left_key, _), (right_key, _)| left_key < right_key) + .peekable(); + + left_slices + .into_iter() + .filter(|s| !s.is_empty()) + .enumerate() + .map(|(i, s)| s.iter().map(move |(k, v)| (i, k, v))) + .kmerge_by(|(i1, k1, _), (i2, k2, _)| (k1, i1) < (k2, i2)) + .dedup_by(|(_, k1, _), (_, k2, _)| *k1 == *k2) + .filter_map(move |(_, key, value)| { + while let Some((right_key, _)) = right_entries.peek().copied() { + if right_key >= key { + break + } + right_entries.next(); + } + + let mut has_mask = false; + let mut has_equal_mask = false; + while let Some((right_key, right_value)) = right_entries.peek().copied() { + if right_key != key { + break + } + + has_mask = true; + if !has_equal_mask { + has_equal_mask = right_value == value; + } + right_entries.next(); + } + + if has_mask && !has_equal_mask { + return None + } + + Some((key.clone(), value.clone())) + }) +} + /// Extend a sorted vector with another sorted vector using 2 pointer merge. /// Values from `other` take precedence for duplicate keys. pub(crate) fn extend_sorted_vec(target: &mut Vec<(K, V)>, other: &[(K, V)]) @@ -81,6 +139,27 @@ where #[cfg(test)] mod tests { use super::*; + use alloc::rc::Rc; + use core::cell::Cell; + + #[derive(Debug)] + struct CloneCounter { + value: u8, + count: Rc>, + } + + impl Clone for CloneCounter { + fn clone(&self) -> Self { + self.count.set(self.count.get() + 1); + Self { value: self.value, count: Rc::clone(&self.count) } + } + } + + impl PartialEq for CloneCounter { + fn eq(&self, other: &Self) -> bool { + self.value == other.value + } + } #[test] fn test_extend_sorted_vec() { @@ -183,4 +262,57 @@ mod tests { let result: Vec<(i32, &str)> = kway_merge_sorted(Vec::<&[(i32, &str)]>::new()); assert!(result.is_empty()); } + + #[test] + fn test_kway_merge_disjoint_sorted() { + let left_old = vec![(1, "old"), (2, "drop"), (4, "keep")]; + let left_new = vec![(1, "new"), (3, "new_only")]; + let right_a = vec![(2, "ignored"), (5, "ignored")]; + let right_b = vec![(3, "ignored")]; + + let result = kway_merge_disjoint_sorted( + [left_new.as_slice(), left_old.as_slice()], + [right_a.as_slice(), right_b.as_slice()], + ) + .collect::>(); + + assert_eq!(result, vec![(1, "new"), (4, "keep")]); + } + + #[test] + fn test_kway_merge_disjoint_sorted_keeps_equal_overlaps() { + let left = vec![(1, "equal"), (2, "equal"), (3, "drop")]; + let right_a = vec![(1, "different"), (2, "equal"), (3, "different")]; + let right_b = vec![(1, "equal"), (2, "different")]; + + let result = + kway_merge_disjoint_sorted([left.as_slice()], [right_a.as_slice(), right_b.as_slice()]) + .collect::>(); + let reversed = + kway_merge_disjoint_sorted([left.as_slice()], [right_b.as_slice(), right_a.as_slice()]) + .collect::>(); + + assert_eq!(result, vec![(1, "equal"), (2, "equal")]); + assert_eq!(reversed, result); + } + + #[test] + fn test_kway_merge_disjoint_sorted_clones_lazily() { + let clone_count = Rc::new(Cell::new(0)); + let left = vec![ + (1, CloneCounter { value: 1, count: Rc::clone(&clone_count) }), + (2, CloneCounter { value: 2, count: Rc::clone(&clone_count) }), + (3, CloneCounter { value: 3, count: Rc::clone(&clone_count) }), + ]; + let mut result = kway_merge_disjoint_sorted( + [left.as_slice()], + core::iter::empty::<&[(i32, CloneCounter)]>(), + ); + + assert_eq!(clone_count.get(), 0); + assert_eq!(result.next().map(|(key, _)| key), Some(1)); + assert_eq!(clone_count.get(), 1); + drop(result); + assert_eq!(clone_count.get(), 1); + } } From e430fd80a974aef7a5fdad90181613e4272905c9 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 30 Jul 2026 16:01:34 +0200 Subject: [PATCH 066/148] fix(net): keep ENR ports paired with their IP address (#26529) Co-authored-by: Chase Wright --- crates/net/dns/src/lib.rs | 17 +++---- crates/net/peers/src/lib.rs | 18 ++------ crates/net/peers/src/node_record.rs | 71 +++++++++++++++++++++++------ 3 files changed, 69 insertions(+), 37 deletions(-) diff --git a/crates/net/dns/src/lib.rs b/crates/net/dns/src/lib.rs index 31b147ed321..fd238e31c9e 100644 --- a/crates/net/dns/src/lib.rs +++ b/crates/net/dns/src/lib.rs @@ -23,12 +23,11 @@ pub use config::DnsDiscoveryConfig; use enr::Enr; pub use error::ParseDnsEntryError; use reth_ethereum_forks::{EnrForkIdEntry, ForkId}; -use reth_network_peers::{pk2id, NodeRecord}; +use reth_network_peers::NodeRecord; use schnellru::{ByLength, LruMap}; use secp256k1::SecretKey; use std::{ collections::{hash_map::Entry, HashMap, HashSet, VecDeque}, - net::IpAddr, pin::Pin, sync::Arc, task::{ready, Context, Poll}, @@ -390,13 +389,8 @@ pub enum DnsDiscoveryEvent { /// Converts an [Enr] into a [`NodeRecord`] fn convert_enr_node_record(enr: &Enr) -> Option { - let node_record = NodeRecord { - address: enr.ip4().map(IpAddr::from).or_else(|| enr.ip6().map(IpAddr::from))?, - tcp_port: enr.tcp4().or_else(|| enr.tcp6())?, - udp_port: enr.udp4().or_else(|| enr.udp6())?, - id: pk2id(&enr.public_key()), - } - .into_ipv4_mapped(); + // DNS discovery yields RLPx dial targets, so records without a tcp endpoint are skipped. + let node_record = NodeRecord::try_from(enr).ok().filter(|record| record.tcp_port != 0)?; let fork_id = enr.get_decodable::(b"eth").transpose().ok().flatten().map(Into::into); @@ -416,7 +410,10 @@ mod tests { use reth_chainspec::MAINNET; use reth_ethereum_forks::{EthereumHardfork, ForkHash}; use secp256k1::rand::thread_rng; - use std::{future::poll_fn, net::Ipv4Addr}; + use std::{ + future::poll_fn, + net::{IpAddr, Ipv4Addr}, + }; fn entry_hash(entry_txt: &str) -> String { BASE32_NOPAD.encode(&keccak256(entry_txt.as_bytes()).as_slice()[..16]) diff --git a/crates/net/peers/src/lib.rs b/crates/net/peers/src/lib.rs index e7a656cb94c..d3bfe8bc64f 100644 --- a/crates/net/peers/src/lib.rs +++ b/crates/net/peers/src/lib.rs @@ -157,23 +157,13 @@ impl AnyNode { } /// Returns the full node record if available. + /// + /// An ENR that advertises no tcp port has no `RLPx` endpoint, so it does not yield a record. #[cfg(feature = "secp256k1")] pub fn node_record(&self) -> Option { match self { Self::NodeRecord(record) => Some(*record), - Self::Enr(enr) => { - let node_record = NodeRecord { - address: enr - .ip4() - .map(core::net::IpAddr::from) - .or_else(|| enr.ip6().map(core::net::IpAddr::from))?, - tcp_port: enr.tcp4().or_else(|| enr.tcp6())?, - udp_port: enr.udp4().or_else(|| enr.udp6())?, - id: pk2id(&enr.public_key()), - } - .into_ipv4_mapped(); - Some(node_record) - } + Self::Enr(enr) => NodeRecord::try_from(enr).ok().filter(|record| record.tcp_port != 0), Self::PeerId(_) | Self::TrustedPeer(_) => None, } } @@ -356,6 +346,8 @@ mod tests { .parse::() .unwrap() ); + // The spec vector is discovery-only (no tcp key), so it has no RLPx endpoint. + assert!(node.node_record().is_none()); assert_eq!(node.to_string(), url); } diff --git a/crates/net/peers/src/node_record.rs b/crates/net/peers/src/node_record.rs index 4b3b5f75af8..8cf25e22414 100644 --- a/crates/net/peers/src/node_record.rs +++ b/crates/net/peers/src/node_record.rs @@ -213,22 +213,24 @@ impl TryFrom<&Enr> for NodeRecord { type Error = NodeRecordParseError; fn try_from(enr: &Enr) -> Result { - let Some(address) = enr.ip4().map(IpAddr::from).or_else(|| enr.ip6().map(IpAddr::from)) - else { - return Err(NodeRecordParseError::InvalidUrl("ip missing".to_string())) - }; - - let Some(udp_port) = enr.udp4().or_else(|| enr.udp6()) else { - return Err(NodeRecordParseError::InvalidUrl("udp port missing".to_string())) - }; - - // A discovery-only ENR (e.g. a devp2p bootnode) omits the tcp key; treat that as no RLPx - // (port 0) rather than rejecting the record. - let tcp_port = enr.tcp4().or_else(|| enr.tcp6()).unwrap_or(0); + let endpoint = enr + .ip4() + .zip(enr.udp4()) + .map(|(ip, udp)| (IpAddr::from(ip), udp, enr.tcp4().unwrap_or(0))); + // Generic `udp` and `tcp` entries also apply to IPv6 when their IPv6-specific counterparts + // are absent. + let (address, udp_port, tcp_port) = endpoint + .or_else(|| { + enr.ip6().zip(enr.udp6().or_else(|| enr.udp4())).map(|(ip, udp)| { + (IpAddr::from(ip), udp, enr.tcp6().or_else(|| enr.tcp4()).unwrap_or(0)) + }) + }) + .ok_or_else(|| { + NodeRecordParseError::InvalidUrl("ip or matching udp port missing".to_string()) + })?; let id = crate::pk2id(&enr.public_key()); - - Ok(Self { address, tcp_port, udp_port, id }.into_ipv4_mapped()) + Ok(Self { address, udp_port, tcp_port, id }.into_ipv4_mapped()) } } @@ -421,4 +423,45 @@ mod tests { assert_eq!(record.tcp_port, 0); assert_eq!(record.udp_port, 30301); } + + #[test] + #[cfg(feature = "secp256k1")] + fn enr_endpoint_keeps_ip_and_ports_together() { + let sk = secp256k1::SecretKey::from_byte_array(&[1u8; 32]).unwrap(); + let mut builder = Enr::builder(); + builder.ip4("10.0.0.1".parse().unwrap()).ip6("::1".parse().unwrap()); + builder.udp6(30401).tcp6(30403); + let enr = builder.build(&sk).unwrap(); + let record = NodeRecord::try_from(&enr).unwrap(); + + assert_eq!(record.address, "::1".parse::().unwrap()); + assert_eq!(record.udp_port, 30401); + assert_eq!(record.tcp_port, 30403); + + // IPv6-only record with the RLPx port only under the generic `tcp` key, the shape geth + // publishes: the generic entry applies to the IPv6 endpoint. + let mut builder = Enr::builder(); + builder.ip6("::1".parse().unwrap()).udp6(30401).tcp4(30303); + let enr = builder.build(&sk).unwrap(); + let record = NodeRecord::try_from(&enr).unwrap(); + + assert_eq!(record.address, "::1".parse::().unwrap()); + assert_eq!(record.udp_port, 30401); + assert_eq!(record.tcp_port, 30303); + + // A full dual-stack record resolves to the IPv4 endpoint. + let mut builder = Enr::builder(); + builder.ip4("10.0.0.1".parse().unwrap()).udp4(30301).tcp4(30303); + builder.ip6("::1".parse().unwrap()).udp6(30401).tcp6(30403); + let enr = builder.build(&sk).unwrap(); + let record = NodeRecord::try_from(&enr).unwrap(); + + assert_eq!(record.address, "10.0.0.1".parse::().unwrap()); + assert_eq!(record.udp_port, 30301); + assert_eq!(record.tcp_port, 30303); + + // No udp port in any family is rejected. + let enr = Enr::builder().ip4("10.0.0.1".parse().unwrap()).tcp4(30303).build(&sk).unwrap(); + assert!(NodeRecord::try_from(&enr).is_err()); + } } From c04c122350b2333ff9711a44494e5a9be78a104b Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 30 Jul 2026 16:52:51 +0200 Subject: [PATCH 067/148] feat(provider): support trie-only save_blocks (#26535) --- crates/engine/tree/src/persistence.rs | 27 ++-- .../src/providers/database/provider.rs | 147 ++++++++++++++---- .../src/providers/database/save_blocks.rs | 45 ++++-- 3 files changed, 159 insertions(+), 60 deletions(-) diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 50ce806a77f..6814c275c22 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -95,24 +95,25 @@ where while let Ok(action) = self.incoming.recv() { match action { PersistenceAction::RemoveBlocksAbove(new_tip_num, sender) => { - let last_block = self.on_remove_blocks_above(new_tip_num)?; + let result = self.on_remove_blocks_above(new_tip_num)?; // send new sync metrics based on removed blocks let _ = self.sync_metrics_tx.send(MetricEvent::SyncHeight { height: new_tip_num }); - let _ = sender.send(PersistenceResult { last_block, commit_duration: None }); + let _ = sender.send(result); } - PersistenceAction::SaveBlocks(blocks, sender) => { - let result = self.on_save_blocks(blocks)?; - let result_number = result.last_block.map(|b| b.number); + PersistenceAction::SaveBlocks(input, sender) => { + let new_db_tip = input.new_db_tip(); + let db_tip_advanced = input.prev_db_tip() < new_db_tip; + let result = self.on_save_blocks(input)?; let _ = sender.send(result); - if let Some(block_number) = result_number { + if db_tip_advanced { // send new sync metrics based on saved blocks let _ = self .sync_metrics_tx - .send(MetricEvent::SyncHeight { height: block_number }); - self.maybe_run_pruner(block_number)?; + .send(MetricEvent::SyncHeight { height: new_db_tip }); + self.maybe_run_pruner(new_db_tip)?; } } PersistenceAction::SaveFinalizedBlock(finalized_block) => { @@ -130,7 +131,7 @@ where fn on_remove_blocks_above( &self, new_tip_num: u64, - ) -> Result, PersistenceError> { + ) -> Result { debug!(target: "engine::persistence", ?new_tip_num, "Removing blocks"); let start_time = Instant::now(); let provider_rw = self.provider.database_provider_rw()?; @@ -141,7 +142,10 @@ where debug!(target: "engine::persistence", ?new_tip_num, ?new_tip_hash, "Removed blocks from disk"); self.metrics.remove_blocks_above_duration_seconds.record(start_time.elapsed()); - Ok(new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num })) + Ok(PersistenceResult { + last_block: new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num }), + commit_duration: None, + }) } #[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = input.persist_rest_blocks().len()))] @@ -149,7 +153,8 @@ where &mut self, input: SaveBlocksInput, ) -> Result { - let first_block = input.first_persist_rest_block().recovered_block().num_hash(); + let first_block = + input.first_persist_rest_block().map(|block| block.recovered_block().num_hash()); let last_block = input.last_block(); let block_count = input.persist_rest_blocks().len(); diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 714f79cfd18..e8133923e35 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -616,38 +616,49 @@ impl DatabaseProvider ProviderResult<()> { let total_start = Instant::now(); let block_count = blocks.len() as u64; - let first_number = blocks.first().unwrap().recovered_block().number(); - let last_block_number = blocks.last().unwrap().recovered_block().number(); + // With no new block data, the masking suffix still ends at the database tip. If the + // masking suffix is empty, the state/trie range ends there instead. + let last_block_number = blocks + .last() + .or_else(|| state_trie_masking_blocks.last()) + .or_else(|| state_trie_blocks.last()) + .expect("at least one persistence range must be non-empty") + .recovered_block() + .number(); + let first_number = blocks.first().map(|block| block.recovered_block().number()); debug!(target: "providers::db", block_count, "Writing blocks and execution data to storage"); // Compute tx_nums upfront (both threads need these) - let first_tx_num = self - .tx - .cursor_read::()? - .last()? - .map(|(n, _)| n + 1) - .unwrap_or_default(); - - let tx_nums: SmallVec<[TxNumber; 4]> = { - let mut nums = SmallVec::with_capacity(blocks.len()); + let mut tx_nums: SmallVec<[TxNumber; 4]> = SmallVec::with_capacity(blocks.len()); + if !blocks.is_empty() { + let first_tx_num = self + .tx + .cursor_read::()? + .last()? + .map(|(n, _)| n + 1) + .unwrap_or_default(); let mut current = first_tx_num; for block in blocks { - nums.push(current); + tx_nums.push(current); current += block.recovered_block().body().transaction_count() as u64; } - nums - }; + } let mut timings = metrics::SaveBlocksTimings { batch_size: block_count, ..Default::default() }; // avoid capturing &self.tx in scope below. let sf_provider = &self.static_file_provider; - let sf_ctx = self.static_file_write_ctx(save_mode, first_number, last_block_number)?; - let rocksdb_provider = self.rocksdb_provider.clone(); - let rocksdb_ctx = self.rocksdb_write_ctx(first_number); - let rocksdb_enabled = rocksdb_ctx.storage_settings.storage_v2; + let rocksdb_provider = &self.rocksdb_provider; + let sf_ctx = first_number + .map(|first_number| { + self.static_file_write_ctx(save_mode, first_number, last_block_number) + }) + .transpose()?; + let rocksdb_ctx = first_number.map(|first_number| self.rocksdb_write_ctx(first_number)); + let rocksdb_enabled = + rocksdb_ctx.as_ref().is_some_and(|ctx| ctx.storage_settings.storage_v2); let mut sf_result = None; let mut rocksdb_result = None; @@ -659,21 +670,27 @@ impl DatabaseProvider DatabaseProvider DatabaseProvider DatabaseProvider DatabaseProvider StageCheckpoint::new(last_block_number) @@ -812,7 +836,9 @@ impl DatabaseProvider DatabaseProvider, _>>() .unwrap(); assert!(masked_entries.is_empty()); + + drop(storage_trie); + drop(account_trie); + drop(hashed_storages); + drop(hashed_accounts); + drop(provider); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new(vec![deferred_trie_block], 2, 1, 2, 2); + assert!(input.persist_rest_blocks().is_empty()); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let finish_checkpoint = provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); + assert_eq!(finish_checkpoint.block_number, 2); + assert!(finish_checkpoint.finish_stage_checkpoint().is_none()); + + let mut hashed_accounts = + provider.tx_ref().cursor_read::().unwrap(); + let (_, account) = hashed_accounts.seek_exact(masked_account).unwrap().unwrap(); + assert_eq!(account.nonce, 3); + + let mut hashed_storages = + provider.tx_ref().cursor_dup_read::().unwrap(); + let storage = + hashed_storages.seek_by_key_subkey(masked_storage, masked_slot).unwrap().unwrap(); + assert_eq!(storage.value, U256::from(4)); + + let mut account_trie = provider.tx_ref().cursor_read::().unwrap(); + assert!(account_trie.seek_exact(StoredNibbles(masked_account_node)).unwrap().is_some()); + + let mut storage_trie = provider.tx_ref().cursor_dup_read::().unwrap(); + let masked_entries: Vec<_> = storage_trie + .walk_dup(Some(masked_storage), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert_eq!(masked_entries.len(), 1); + assert_eq!(masked_entries[0].1.nibbles.0, masked_storage_node); } #[cfg(feature = "partial-persistence")] diff --git a/crates/storage/provider/src/providers/database/save_blocks.rs b/crates/storage/provider/src/providers/database/save_blocks.rs index 66674f3ff54..c21bb152a17 100644 --- a/crates/storage/provider/src/providers/database/save_blocks.rs +++ b/crates/storage/provider/src/providers/database/save_blocks.rs @@ -16,7 +16,8 @@ use reth_primitives_traits::NodePrimitives; /// `blocks` contains every canonical block in `(prev_partial_state_trie, new_db_tip]`, ordered from /// oldest to newest. The four frontiers derive the ranges written during this operation: /// -/// - `(prev_db_tip, new_db_tip]`: persist block, execution, and history data; +/// - `(prev_db_tip, new_db_tip]`: persist block, execution, and history data, if the database +/// frontier advances; /// - `(prev_partial_state_trie, new_partial_state_trie]`: consider hashed-state/trie updates for /// persistence; and /// - `(new_partial_state_trie, new_db_tip]`: retain as the masking suffix and do not persist its @@ -41,14 +42,14 @@ impl SaveBlocksInput { /// Creates an input that advances the existing frontiers to `new_db_tip` and /// `new_partial_state_trie`. /// - /// `new_partial_state_trie` may remain behind `prev_db_tip`. This occurs while a masking - /// window is first growing: new block data advances without forcing all previously masked - /// hashed-state/trie updates to disk. + /// Either frontier may advance independently. `new_partial_state_trie` may remain behind + /// `prev_db_tip` while a masking window grows, and `new_db_tip` may remain unchanged while the + /// state/trie frontier catches up. /// /// # Panics /// - /// Panics if the database frontier does not advance, the state/trie frontier moves backwards - /// or exceeds the database frontier, or `blocks` is not the exact contiguous range + /// Panics if either frontier moves backwards, neither frontier advances, the state/trie + /// frontier exceeds the database frontier, or `blocks` is not the exact contiguous range /// `(prev_partial_state_trie, new_db_tip]`. pub fn new( blocks: Vec>, @@ -61,11 +62,15 @@ impl SaveBlocksInput { prev_partial_state_trie <= prev_db_tip, "previous state/trie tip must not exceed previous database tip" ); - assert!(prev_db_tip < new_db_tip, "database tip must advance"); + assert!(prev_db_tip <= new_db_tip, "database tip must not move backwards"); assert!( prev_partial_state_trie <= new_partial_state_trie, "state/trie tip must not move backwards" ); + assert!( + prev_db_tip < new_db_tip || prev_partial_state_trie < new_partial_state_trie, + "at least one persistence frontier must advance" + ); assert!( new_partial_state_trie <= new_db_tip, "new state/trie tip must not exceed new database tip" @@ -116,11 +121,9 @@ impl SaveBlocksInput { .num_hash() } - /// Returns the first newly persisted block. - pub fn first_persist_rest_block(&self) -> &ExecutedBlock { - self.persist_rest_blocks() - .first() - .expect("constructor ensures a non-empty persistence range") + /// Returns the first block whose block, execution, and history data will be persisted. + pub fn first_persist_rest_block(&self) -> Option<&ExecutedBlock> { + self.persist_rest_blocks().first() } /// Returns newly persisted blocks whose block, execution, and history data should be written. @@ -185,7 +188,7 @@ mod tests { let blocks = TestBlockBuilder::eth().get_executed_blocks(13..17).collect(); let input = SaveBlocksInput::new(blocks, 15, 12, 16, 13); - assert_eq!(input.first_persist_rest_block().recovered_block().number(), 16); + assert_eq!(input.first_persist_rest_block().unwrap().recovered_block().number(), 16); assert_eq!(input.state_trie_blocks()[0].recovered_block().number(), 13); assert_eq!( input @@ -198,10 +201,20 @@ mod tests { } #[test] - #[should_panic(expected = "database tip must advance")] - fn requires_database_tip_to_advance() { + fn advances_only_state_trie_frontier() { let blocks = TestBlockBuilder::eth().get_executed_blocks(1..2).collect(); - let _ = SaveBlocksInput::new(blocks, 1, 0, 1, 1); + let input = SaveBlocksInput::new(blocks, 1, 0, 1, 1); + + assert!(input.persist_rest_blocks().is_empty()); + assert!(input.first_persist_rest_block().is_none()); + assert_eq!(input.state_trie_blocks()[0].recovered_block().number(), 1); + assert!(input.state_trie_masking_blocks().is_empty()); + } + + #[test] + #[should_panic(expected = "at least one persistence frontier must advance")] + fn requires_a_frontier_to_advance() { + let _ = SaveBlocksInput::::new(vec![], 1, 1, 1, 1); } #[test] From f2e2b5c56593359c9626caf11e392c4f75ddede7 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 30 Jul 2026 17:49:06 +0200 Subject: [PATCH 068/148] refactor(net): add NodeRecord::has_rlpx_endpoint helper (#26538) --- crates/net/dns/src/lib.rs | 2 +- crates/net/peers/src/lib.rs | 2 +- crates/net/peers/src/node_record.rs | 8 ++++++++ crates/net/peers/src/trusted_peer.rs | 2 +- 4 files changed, 11 insertions(+), 3 deletions(-) diff --git a/crates/net/dns/src/lib.rs b/crates/net/dns/src/lib.rs index fd238e31c9e..bac285524d6 100644 --- a/crates/net/dns/src/lib.rs +++ b/crates/net/dns/src/lib.rs @@ -390,7 +390,7 @@ pub enum DnsDiscoveryEvent { /// Converts an [Enr] into a [`NodeRecord`] fn convert_enr_node_record(enr: &Enr) -> Option { // DNS discovery yields RLPx dial targets, so records without a tcp endpoint are skipped. - let node_record = NodeRecord::try_from(enr).ok().filter(|record| record.tcp_port != 0)?; + let node_record = NodeRecord::try_from(enr).ok().filter(NodeRecord::has_rlpx_endpoint)?; let fork_id = enr.get_decodable::(b"eth").transpose().ok().flatten().map(Into::into); diff --git a/crates/net/peers/src/lib.rs b/crates/net/peers/src/lib.rs index d3bfe8bc64f..959fae8da08 100644 --- a/crates/net/peers/src/lib.rs +++ b/crates/net/peers/src/lib.rs @@ -163,7 +163,7 @@ impl AnyNode { pub fn node_record(&self) -> Option { match self { Self::NodeRecord(record) => Some(*record), - Self::Enr(enr) => NodeRecord::try_from(enr).ok().filter(|record| record.tcp_port != 0), + Self::Enr(enr) => NodeRecord::try_from(enr).ok().filter(NodeRecord::has_rlpx_endpoint), Self::PeerId(_) | Self::TrustedPeer(_) => None, } } diff --git a/crates/net/peers/src/node_record.rs b/crates/net/peers/src/node_record.rs index 8cf25e22414..891d253f811 100644 --- a/crates/net/peers/src/node_record.rs +++ b/crates/net/peers/src/node_record.rs @@ -117,6 +117,14 @@ impl NodeRecord { pub const fn udp_addr(&self) -> SocketAddr { SocketAddr::new(self.address, self.udp_port) } + + /// Returns `true` if this record advertises an `RLPx` endpoint, i.e. a non-zero tcp port. + /// + /// Discovery-only records, such as ENRs without a tcp key, convert with a tcp port of 0. + #[must_use] + pub const fn has_rlpx_endpoint(&self) -> bool { + self.tcp_port != 0 + } } impl fmt::Display for NodeRecord { diff --git a/crates/net/peers/src/trusted_peer.rs b/crates/net/peers/src/trusted_peer.rs index 15cc3676e9b..6eef50e714c 100644 --- a/crates/net/peers/src/trusted_peer.rs +++ b/crates/net/peers/src/trusted_peer.rs @@ -121,7 +121,7 @@ impl FromStr for TrustedPeer { let enr = enr::Enr::::from_str(s) .map_err(NodeRecordParseError::InvalidUrl)?; let mut record = NodeRecord::try_from(&enr)?; - if record.tcp_port == 0 { + if !record.has_rlpx_endpoint() { // Discovery-only ENRs commonly share their UDP port with the RLPx listener. record.tcp_port = record.udp_port; } From 9d25749e3182bcefabb9920179bc91e64c4291d3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Thu, 30 Jul 2026 22:36:22 +0200 Subject: [PATCH 069/148] chore: update ruint to 1.20.0 (#26541) --- Cargo.lock | 81 ++++++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 78 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3ecccc2b1d8..95fba14dc24 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1192,6 +1192,23 @@ dependencies = [ "zeroize", ] +[[package]] +name = "ark-ff" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7a806ac6c8307b929df4645776290a50ee2aac754ad09d8bdf73391309e43af" +dependencies = [ + "ark-ff-asm 0.6.0", + "ark-ff-macros 0.6.0", + "ark-serialize 0.6.0", + "ark-std 0.6.0", + "digest 0.10.7", + "educe", + "num-bigint", + "num-traits", + "zeroize", +] + [[package]] name = "ark-ff-asm" version = "0.3.0" @@ -1222,6 +1239,16 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "ark-ff-asm" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1479009684adc073dff49a1025d3a7065b317a9ead25aaaca38cdc70058ba8a2" +dependencies = [ + "quote", + "syn 2.0.118", +] + [[package]] name = "ark-ff-macros" version = "0.3.0" @@ -1260,6 +1287,19 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "ark-ff-macros" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a0691ed21ef00ef89c1e9bda832eba493dda3ec2f8d892fb25b705f73f06bb8" +dependencies = [ + "num-bigint", + "num-traits", + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "ark-poly" version = "0.5.0" @@ -1331,13 +1371,26 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" dependencies = [ - "ark-serialize-derive", + "ark-serialize-derive 0.5.0", "ark-std 0.5.0", "arrayvec", "digest 0.10.7", "num-bigint", ] +[[package]] +name = "ark-serialize" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a74dd304fd536fb95d0a328e72be759209cc496a9da094c5bc56e5fea4f9e86b" +dependencies = [ + "ark-serialize-derive 0.6.0", + "ark-std 0.6.0", + "digest 0.10.7", + "num-bigint", + "serde_with", +] + [[package]] name = "ark-serialize-derive" version = "0.5.0" @@ -1349,6 +1402,17 @@ dependencies = [ "syn 2.0.118", ] +[[package]] +name = "ark-serialize-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f153690697a2b91e5e1251ff98411ee5371500a111a0fd317a70e588eb300f9" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.118", +] + [[package]] name = "ark-std" version = "0.3.0" @@ -1379,6 +1443,16 @@ dependencies = [ "rand 0.8.6", ] +[[package]] +name = "ark-std" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "367c9c827ed431bff6868b7aa926e05b16eb46603cc8b6e768e4a5553fa1d155" +dependencies = [ + "num-traits", + "rand 0.8.6", +] + [[package]] name = "arrayref" version = "0.3.9" @@ -11157,15 +11231,16 @@ checksum = "afab94fb28594581f62d981211a9a4d53cc8130bbcbbb89a0440d9b8e81a7746" [[package]] name = "ruint" -version = "1.18.0" +version = "1.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0298da754d1395046b0afdc2f20ee76d29a8ae310cd30ffa84ed42acba9cb12a" +checksum = "f5e99bff0393163bb25029a6af25d3d8d202ba5b5438a74d1bd8789f5c822970" dependencies = [ "alloy-rlp", "arbitrary", "ark-ff 0.3.0", "ark-ff 0.4.2", "ark-ff 0.5.0", + "ark-ff 0.6.0", "bytes", "fastrlp 0.3.1", "fastrlp 0.4.0", From fd1c9982c9c2ac3bb8782ec5d3f5c030d01e7a56 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Fri, 31 Jul 2026 15:08:50 +0530 Subject: [PATCH 070/148] fix(rpc): cap implicit simulate gas limit (#26502) --- crates/ethereum/node/tests/e2e/simulate.rs | 79 +++++++++++++++++++++- crates/rpc/rpc-eth-api/src/helpers/call.rs | 9 ++- 2 files changed, 84 insertions(+), 4 deletions(-) diff --git a/crates/ethereum/node/tests/e2e/simulate.rs b/crates/ethereum/node/tests/e2e/simulate.rs index cd9d2b1de23..7e6b04af72d 100644 --- a/crates/ethereum/node/tests/e2e/simulate.rs +++ b/crates/ethereum/node/tests/e2e/simulate.rs @@ -1,12 +1,13 @@ -use crate::utils::eth_payload_attributes; +use crate::utils::{eth_payload_attributes, eth_payload_attributes_amsterdam}; use alloy_primitives::{bytes, Address, U256}; use alloy_provider::{network::EthereumWallet, Provider, ProviderBuilder}; +use alloy_rpc_types_engine::PayloadAttributes; use alloy_rpc_types_eth::{ simulate::{SimBlock, SimulatePayload, SimulatedBlock}, state::{AccountOverride, StateOverride, StateOverridesBuilder}, BlockOverrides, TransactionRequest, TransactionTrait as _, }; -use reth_chainspec::{ChainSpecBuilder, MAINNET}; +use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::setup_engine; use reth_node_ethereum::EthereumNode; use std::sync::Arc; @@ -99,6 +100,80 @@ async fn test_simulate_v1_no_fields_call_defaults_to_remaining_block_gas() -> ey Ok(()) } +async fn assert_validation_uses_remaining_gas( + chain_spec: Arc, + payload_attributes: fn(u64) -> PayloadAttributes, +) -> eyre::Result<()> { + reth_tracing::init_test_tracing(); + + let (mut nodes, wallet) = + setup_engine::(1, chain_spec, false, Default::default(), payload_attributes) + .await?; + let node = nodes.pop().unwrap(); + let provider = ProviderBuilder::new() + .wallet(EthereumWallet::new(wallet.wallet_gen().swap_remove(0))) + .connect_http(node.rpc_url()); + + let from: Address = "0xc000000000000000000000000000000000000000".parse()?; + let state_overrides = + StateOverridesBuilder::default().with_balance(from, U256::from(1_000_000_000u64)).build(); + let tx = TransactionRequest::default() + .from(from) + .max_fee_per_gas(0) + .max_priority_fee_per_gas(0) + .nonce(0) + // Create an empty contract so Amsterdam exercises the state-gas reservoir. + .input(bytes!("0x60006000f3").into()); + let sim_block = SimBlock::default() + .with_block_overrides(BlockOverrides { + gas_limit: Some(30_000_000), + base_fee: Some(U256::ZERO), + ..Default::default() + }) + .with_state_overrides(state_overrides) + .call(tx); + let payload = + SimulatePayload::default().with_validation().with_full_transactions().extend(sim_block); + + let result: Vec = + provider.raw_request("eth_simulateV1".into(), (&payload, "latest")).await?; + + assert_eq!(result.len(), 1); + assert_eq!(result[0].calls.len(), 1); + assert!(result[0].calls[0].status); + let txs = result[0].inner.transactions.as_transactions().expect("expected full transactions"); + assert_eq!(txs.len(), 1); + assert_eq!(txs[0].gas_limit(), 30_000_000); + assert!(txs[0].gas_limit() > (1 << 24)); + + Ok(()) +} + +#[tokio::test] +async fn test_simulate_v1_validation_omitted_gas_uses_remaining_gas_osaka() -> eyre::Result<()> { + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .osaka_activated() + .build(), + ); + assert_validation_uses_remaining_gas(chain_spec, eth_payload_attributes).await +} + +#[tokio::test] +async fn test_simulate_v1_validation_omitted_gas_uses_remaining_gas_amsterdam() -> eyre::Result<()> +{ + let chain_spec = Arc::new( + ChainSpecBuilder::default() + .chain(MAINNET.chain) + .genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap()) + .amsterdam_activated() + .build(), + ); + assert_validation_uses_remaining_gas(chain_spec, eth_payload_attributes_amsterdam).await +} + #[tokio::test] async fn test_simulate_v1_blockhash_reads_prior_simulated_block() -> eyre::Result<()> { reth_tracing::init_test_tracing(); diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index e4cccad806b..58431b19d39 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -41,7 +41,7 @@ use reth_rpc_eth_types::{ use reth_storage_api::{BlockIdReader, ProviderTx, StateProviderBox}; use revm::{ context::Block, - context_interface::{result::ResultAndState, Transaction}, + context_interface::{result::ResultAndState, Cfg, Transaction}, Database, DatabaseCommit, }; use revm_inspectors::{access_list::AccessListInspector, transfer::TransferInspector}; @@ -144,11 +144,16 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA // Always disable EIP-3607 evm_env.cfg_env.disable_eip3607 = true; + // EIP-7825's transaction gas cap is only active with Amsterdam's + // regular/state-gas accounting. + if !evm_env.cfg_env.is_amsterdam_eip8037_enabled() { + evm_env.cfg_env.tx_gas_limit_cap = Some(u64::MAX); + } + if !validation { // If not explicitly required, we disable nonce check evm_env.cfg_env.disable_nonce_check = true; evm_env.cfg_env.disable_base_fee = true; - evm_env.cfg_env.tx_gas_limit_cap = Some(u64::MAX); evm_env.block_env.inner_mut().basefee = 0; } From d3519668cbf31fc9c878bc98d94310fa33d7a18c Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 31 Jul 2026 12:13:38 +0200 Subject: [PATCH 071/148] feat(engine): configure state masking blocks (#26533) --- crates/chain-state/src/in_memory.rs | 17 +- crates/engine/primitives/src/config.rs | 89 ++++++++- crates/engine/tree/src/persistence.rs | 71 +++++-- crates/engine/tree/src/tree/mod.rs | 186 ++++++++++-------- .../engine/tree/src/tree/persistence_state.rs | 23 ++- crates/engine/tree/src/tree/tests.rs | 130 ++++++++++-- crates/node/core/src/args/engine.rs | 91 ++++++++- .../src/providers/database/provider.rs | 80 +++++++- .../storage/storage-api/src/block_writer.rs | 23 ++- 9 files changed, 566 insertions(+), 144 deletions(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index d9028830656..41e36848e60 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -319,6 +319,16 @@ impl CanonicalInMemoryState { /// This will update the links between blocks and remove all blocks that are [.. /// `persisted_height`]. pub fn remove_persisted_blocks(&self, persisted_num_hash: BlockNumHash) { + self.remove_persisted_blocks_until(persisted_num_hash, persisted_num_hash.number); + } + + /// Removes blocks from the in-memory state through `remove_until` while still reporting the + /// provided block as the persisted tip. + pub fn remove_persisted_blocks_until( + &self, + persisted_num_hash: BlockNumHash, + remove_until: BlockNumber, + ) { self.set_persisted(persisted_num_hash); // if the persisted hash is not in the canonical in memory state, do nothing, because it // means canonical blocks were not actually persisted. @@ -336,16 +346,15 @@ impl CanonicalInMemoryState { let mut numbers = self.inner.in_memory_state.numbers.write(); let mut blocks = self.inner.in_memory_state.blocks.write(); - let BlockNumHash { number: persisted_height, hash: _ } = persisted_num_hash; + let remove_until = remove_until.min(persisted_num_hash.number); // clear all numbers numbers.clear(); - // drain all blocks and only keep the ones that are not persisted (below the persisted - // height) + // Drain all blocks and keep only the suffix that still has to stay in memory. let mut old_blocks = blocks .drain() - .filter(|(_, b)| b.block_ref().recovered_block().number() > persisted_height) + .filter(|(_, b)| b.block_ref().recovered_block().number() > remove_until) .map(|(_, b)| b.block.clone()) .collect::>(); diff --git a/crates/engine/primitives/src/config.rs b/crates/engine/primitives/src/config.rs index 108db6ee47e..7bf54a66f5f 100644 --- a/crates/engine/primitives/src/config.rs +++ b/crates/engine/primitives/src/config.rs @@ -6,6 +6,9 @@ use core::time::Duration; /// Triggers persistence when the number of canonical blocks in memory exceeds this threshold. pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 7; +/// Number of persisted blocks whose state/trie writes are masked by an in-memory suffix. +pub const DEFAULT_NUM_STATE_MASKING_BLOCKS: u64 = 0; + /// Maximum number of blocks beyond the in-memory buffer target awaiting persistence before engine /// API processing is stalled. pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 16; @@ -51,6 +54,21 @@ const fn assert_backpressure_threshold_invariant( ); } +const fn assert_state_masking_invariant( + persistence_threshold: u64, + num_state_masking_blocks: u64, + memory_block_buffer_target: u64, +) { + let valid_window = match num_state_masking_blocks.checked_add(memory_block_buffer_target) { + Some(window) => window < persistence_threshold, + None => false, + }; + debug_assert!( + num_state_masking_blocks == 0 || valid_window, + "num_state_masking_blocks + memory_block_buffer_target must be less than persistence_threshold", + ); +} + const fn default_cross_block_cache_size() -> usize { if cfg!(test) { 1024 * 1024 // 1 MB in tests @@ -84,6 +102,9 @@ pub struct TreeConfig { /// Maximum number of blocks to be kept only in memory without triggering /// persistence. persistence_threshold: u64, + /// Number of persisted blocks whose state/trie writes are masked instead of being durably + /// written in the current cycle. + num_state_masking_blocks: u64, /// How close to the canonical head we persist blocks. Represents the ideal /// number of most recent blocks to keep in memory for quick access and reorgs. /// @@ -207,8 +228,14 @@ impl Default for TreeConfig { DEFAULT_PERSISTENCE_THRESHOLD, DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, ); + assert_state_masking_invariant( + DEFAULT_PERSISTENCE_THRESHOLD, + DEFAULT_NUM_STATE_MASKING_BLOCKS, + DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, + ); Self { persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, + num_state_masking_blocks: DEFAULT_NUM_STATE_MASKING_BLOCKS, memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, block_buffer_limit: DEFAULT_BLOCK_BUFFER_LIMIT, @@ -251,6 +278,7 @@ impl TreeConfig { #[expect(clippy::too_many_arguments)] pub const fn new( persistence_threshold: u64, + num_state_masking_blocks: u64, memory_block_buffer_target: u64, persistence_backpressure_threshold: u64, block_buffer_limit: u32, @@ -280,8 +308,14 @@ impl TreeConfig { persistence_threshold, persistence_backpressure_threshold, ); + assert_state_masking_invariant( + persistence_threshold, + num_state_masking_blocks, + memory_block_buffer_target, + ); Self { persistence_threshold, + num_state_masking_blocks, memory_block_buffer_target, persistence_backpressure_threshold, block_buffer_limit, @@ -323,6 +357,11 @@ impl TreeConfig { self.persistence_threshold } + /// Return the number of persisted blocks whose state/trie writes are masked. + pub const fn num_state_masking_blocks(&self) -> u64 { + self.num_state_masking_blocks + } + /// Return the memory block buffer target. pub const fn memory_block_buffer_target(&self) -> u64 { self.memory_block_buffer_target @@ -440,6 +479,22 @@ impl TreeConfig { self.persistence_threshold, self.persistence_backpressure_threshold, ); + assert_state_masking_invariant( + self.persistence_threshold, + self.num_state_masking_blocks, + self.memory_block_buffer_target, + ); + self + } + + /// Setter for the number of persisted blocks whose state/trie writes are masked. + pub const fn with_num_state_masking_blocks(mut self, num_state_masking_blocks: u64) -> Self { + self.num_state_masking_blocks = num_state_masking_blocks; + assert_state_masking_invariant( + self.persistence_threshold, + self.num_state_masking_blocks, + self.memory_block_buffer_target, + ); self } @@ -449,6 +504,11 @@ impl TreeConfig { memory_block_buffer_target: u64, ) -> Self { self.memory_block_buffer_target = memory_block_buffer_target; + assert_state_masking_invariant( + self.persistence_threshold, + self.num_state_masking_blocks, + self.memory_block_buffer_target, + ); self } @@ -755,7 +815,7 @@ impl TreeConfig { #[cfg(test)] mod tests { - use super::TreeConfig; + use super::{TreeConfig, DEFAULT_NUM_STATE_MASKING_BLOCKS}; #[test] fn txpool_prewarming_is_disabled_by_default_and_can_be_enabled() { @@ -786,4 +846,31 @@ mod tests { .with_persistence_threshold(4) .with_persistence_backpressure_threshold(4); } + + #[test] + fn state_masking_is_disabled_by_default() { + assert_eq!( + TreeConfig::default().num_state_masking_blocks(), + DEFAULT_NUM_STATE_MASKING_BLOCKS + ); + } + + #[test] + #[should_panic( + expected = "num_state_masking_blocks + memory_block_buffer_target must be less than persistence_threshold" + )] + fn rejects_state_masking_window_at_or_above_persistence_threshold() { + let _ = TreeConfig::default() + .with_persistence_threshold(4) + .with_memory_block_buffer_target(2) + .with_num_state_masking_blocks(2); + } + + #[test] + #[should_panic( + expected = "num_state_masking_blocks + memory_block_buffer_target must be less than persistence_threshold" + )] + fn rejects_overflowing_state_masking_window() { + let _ = TreeConfig::default().with_num_state_masking_blocks(u64::MAX); + } } diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 6814c275c22..7f461b9e461 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -25,8 +25,10 @@ use tracing::{debug, error, instrument, warn}; /// Unified result of any persistence operation. #[derive(Debug)] pub struct PersistenceResult { - /// The last block that was persisted, if any. - pub last_block: Option, + /// The highest block whose non-state/trie outputs are persisted. + pub last_block: BlockNumHash, + /// The state/trie persistence frontier. + pub last_state_trie_block: BlockNumHash, /// The commit duration, only available for save-blocks operations. pub commit_duration: Option, } @@ -136,16 +138,25 @@ where let start_time = Instant::now(); let provider_rw = self.provider.database_provider_rw()?; - let new_tip_hash = provider_rw.block_hash(new_tip_num)?; - provider_rw.remove_block_and_execution_above(new_tip_num)?; + let new_tip_hash = provider_rw + .block_hash(new_tip_num)? + .ok_or_else(|| ProviderError::HeaderNotFound(new_tip_num.into()))?; + let frontiers = provider_rw.remove_block_and_execution_above(new_tip_num)?; + debug_assert_eq!(frontiers.db_tip, new_tip_num); + let last_block = BlockNumHash::new(new_tip_num, new_tip_hash); + let last_state_trie_block = if frontiers.partial_state_trie == new_tip_num { + last_block + } else { + let hash = provider_rw.block_hash(frontiers.partial_state_trie)?.ok_or_else(|| { + ProviderError::HeaderNotFound(frontiers.partial_state_trie.into()) + })?; + BlockNumHash::new(frontiers.partial_state_trie, hash) + }; provider_rw.commit()?; debug!(target: "engine::persistence", ?new_tip_num, ?new_tip_hash, "Removed blocks from disk"); self.metrics.remove_blocks_above_duration_seconds.record(start_time.elapsed()); - Ok(PersistenceResult { - last_block: new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num }), - commit_duration: None, - }) + Ok(PersistenceResult { last_block, last_state_trie_block, commit_duration: None }) } #[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = input.persist_rest_blocks().len()))] @@ -166,6 +177,19 @@ where let start_time = Instant::now(); let provider_rw = self.provider.database_provider_rw()?; + let last_state_trie_block = if let Some(block) = input.state_trie_blocks().last() { + // Newly written static-file headers are not readable until commit finalizes their + // index. + block.recovered_block().num_hash() + } else { + // If the state/trie frontier did not advance, its block is excluded from + // `state_trie_blocks()` and must be loaded from already-persisted storage. + let number = input.new_partial_state_trie(); + let hash = provider_rw + .block_hash(number)? + .ok_or_else(|| ProviderError::HeaderNotFound(number.into()))?; + BlockNumHash::new(number, hash) + }; provider_rw.save_blocks(&input)?; if let Some(finalized) = pending_finalized { @@ -191,7 +215,7 @@ where self.metrics.save_blocks_batch_size.record(block_count as f64); self.metrics.save_blocks_duration_seconds.record(elapsed); - Ok(PersistenceResult { last_block: Some(last_block), commit_duration: Some(elapsed) }) + Ok(PersistenceResult { last_block, last_state_trie_block, commit_duration: Some(elapsed) }) } fn maybe_run_pruner(&mut self, block_number: u64) -> Result<(), PersistenceError> { @@ -475,6 +499,25 @@ mod tests { service.maybe_run_pruner(2).unwrap(); } + #[test] + fn test_remove_blocks_above_requires_tip_header() { + let provider = create_test_provider_factory(); + init_genesis(&provider).unwrap(); + + let (_finished_exex_height_tx, finished_exex_height_rx) = + tokio::sync::watch::channel(FinishedExExHeight::NoExExs); + let pruner = + Pruner::new_with_factory(provider.clone(), vec![], 5, 0, None, finished_exex_height_rx); + let (_db_service_tx, db_service_rx) = std::sync::mpsc::channel(); + let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); + let service = PersistenceService::new(provider, db_service_rx, pruner, sync_metrics_tx); + + assert!(matches!( + service.on_remove_blocks_above(1), + Err(PersistenceError::ProviderError(ProviderError::HeaderNotFound(_))) + )); + } + #[derive(Debug)] struct FailingPruneBalStore; @@ -513,7 +556,8 @@ mod tests { let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out"); - assert_eq!(block_hash, result.last_block.unwrap().hash); + assert_eq!(block_hash, result.last_block.hash); + assert_eq!(result.last_state_trie_block, result.last_block); } #[test] @@ -528,7 +572,7 @@ mod tests { handle.save_blocks(full_save_input(blocks), tx).unwrap(); let result = rx.recv().unwrap(); - assert_eq!(last_hash, result.last_block.unwrap().hash); + assert_eq!(last_hash, result.last_block.hash); } #[test] @@ -546,7 +590,7 @@ mod tests { handle.save_blocks(full_save_input(blocks), tx).unwrap(); let result = rx.recv().unwrap(); - assert_eq!(last_hash, result.last_block.unwrap().hash); + assert_eq!(last_hash, result.last_block.hash); } } @@ -689,7 +733,8 @@ mod tests { let pf = provider_factory.clone(); let reorg_handle = std::thread::spawn(move || { let provider_rw = pf.database_provider_rw().unwrap(); - provider_rw.remove_block_and_execution_above(1).unwrap(); + let frontiers = provider_rw.remove_block_and_execution_above(1).unwrap(); + assert_eq!(frontiers.partial_state_trie, 1); provider_rw.commit().unwrap(); let provider_rw = pf.database_provider_rw().unwrap(); diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 8fae1c7410a..d88d260540d 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -475,6 +475,7 @@ where let persistence_state = PersistenceState { last_persisted_block: BlockNumHash::new(best_block_number, header.hash()), + last_state_trie_persisted_block: BlockNumHash::new(best_block_number, header.hash()), rx: None, }; @@ -1415,28 +1416,9 @@ where /// Helper method to save blocks and set the persistence state. This ensures we keep track of /// the current persistence action while we're saving blocks. - fn persist_blocks(&mut self, blocks_to_persist: Vec>) { - if blocks_to_persist.is_empty() { - debug!(target: "engine::tree", "Returned empty set of blocks to persist"); - return - } - - // NOTE: checked non-empty above - let highest_num_hash = blocks_to_persist - .iter() - .max_by_key(|block| block.recovered_block().number()) - .map(|b| b.recovered_block().num_hash()) - .expect("Checked non-empty persisting blocks"); - debug!(target: "engine::tree", count=blocks_to_persist.len(), blocks = ?blocks_to_persist.iter().map(|block| block.recovered_block().num_hash()).collect::>(), "Persisting blocks"); - - let previous_tip = self.persistence_state.last_persisted_block.number; - let input = SaveBlocksInput::new( - blocks_to_persist, - previous_tip, - previous_tip, - highest_num_hash.number, - highest_num_hash.number, - ); + fn persist_blocks(&mut self, input: SaveBlocksInput) { + let highest_num_hash = input.last_block(); + debug!(target: "engine::tree", count=input.persist_rest_blocks().len(), blocks = ?input.persist_rest_blocks().iter().map(|block| block.recovered_block().num_hash()).collect::>(), "Persisting blocks"); let (tx, rx) = crossbeam_channel::bounded(1); let _ = self.persistence.save_blocks(input, tx); @@ -1452,10 +1434,8 @@ where if !self.persistence_state.in_progress() { if let Some(new_tip_num) = self.find_disk_reorg()? { self.remove_blocks(new_tip_num) - } else if self.should_persist() { - let blocks_to_persist = - self.get_canonical_blocks_to_persist(PersistTarget::Threshold)?; - self.persist_blocks(blocks_to_persist); + } else if let Some(input) = self.get_save_blocks_input(PersistTarget::Threshold) { + self.persist_blocks(input); } } @@ -1484,17 +1464,24 @@ where debug!(target: "engine::tree", ?action, "waiting for in-flight persistence"); let result = rx.recv().map_err(|_| AdvancePersistenceError::ChannelClosed)?; self.on_persistence_complete(result, start_time)?; + continue } - let blocks_to_persist = self.get_canonical_blocks_to_persist(PersistTarget::Head)?; + // Persistence can finish against a branch that the in-memory canonical chain has + // reorged away from. Unwind that stale disk branch before building a head-targeted + // save. + if let Some(new_tip_num) = self.find_disk_reorg()? { + self.remove_blocks(new_tip_num); + continue + } - if blocks_to_persist.is_empty() { + let Some(input) = self.get_save_blocks_input(PersistTarget::Head) else { debug!(target: "engine::tree", "persistence complete, signaling termination"); return Ok(()) - } + }; - debug!(target: "engine::tree", count = blocks_to_persist.len(), "persisting remaining blocks before shutdown"); - self.persist_blocks(blocks_to_persist); + debug!(target: "engine::tree", count = input.persist_rest_blocks().len(), "persisting remaining blocks before shutdown"); + self.persist_blocks(input); } } @@ -1531,18 +1518,16 @@ where self.metrics.engine.persistence_duration.record(start_time.elapsed()); let commit_duration = result.commit_duration; - let Some(BlockNumHash { - hash: last_persisted_block_hash, - number: last_persisted_block_number, - }) = result.last_block - else { - // if this happened, then we persisted no blocks because we sent an empty vec of blocks - warn!(target: "engine::tree", "Persistence task completed but did not persist any blocks"); - return Ok(()) - }; + let last_persisted_block = result.last_block; + let last_persisted_block_number = last_persisted_block.number; + let last_state_trie_persisted_block = result.last_state_trie_block; + debug_assert!( + last_state_trie_persisted_block.number <= last_persisted_block.number, + "state/trie frontier cannot exceed the last persisted block" + ); - debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish"); - self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number); + debug!(target: "engine::tree", ?last_persisted_block, ?last_state_trie_persisted_block, elapsed=?start_time.elapsed(), "Finished persisting, calling finish"); + self.persistence_state.finish(last_persisted_block, last_state_trie_persisted_block); // Evict cached changesets for blocks below the eviction threshold. // Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect @@ -1895,7 +1880,7 @@ where // update the tracked chain height, after backfill sync both the canonical height and // persisted height are the same self.state.tree_state.set_canonical_head(new_head.num_hash()); - self.persistence_state.finish(new_head.hash(), new_head.number()); + self.persistence_state.finish(new_head.num_hash(), new_head.num_hash()); // update the tracked canonical head self.canonical_in_memory_state.set_canonical_head(new_head); @@ -2113,70 +2098,95 @@ where ); } - /// Returns true if the canonical chain length minus the last persisted - /// block is greater than the persistence threshold, - /// backfill is not running, and no payload is currently being built. - pub const fn should_persist(&self) -> bool { - if self.building_payload { - return false - } - - if !self.backfill_sync_state.is_idle() { - // can't persist if backfill is running - return false - } - - let min_block = self.persistence_state.last_persisted_block.number; - self.state.tree_state.canonical_block_number().saturating_sub(min_block) > - self.config.persistence_threshold() - } - - /// Returns a batch of consecutive canonical blocks to persist in the range - /// `(last_persisted_number .. target]`. The expected order is oldest -> newest. - fn get_canonical_blocks_to_persist( - &self, - target: PersistTarget, - ) -> Result>, AdvancePersistenceError> { + /// Returns the blocks and frontiers for the next persistence cycle, if one should start. + /// + /// Threshold persistence honors the normal scheduling gates and retains the configured + /// in-memory block buffer. Head persistence bypasses those gates during shutdown and returns + /// `None` once both persistence frontiers have reached the canonical head. + fn get_save_blocks_input(&self, target: PersistTarget) -> Option> { // We will calculate the state root using the database, so we need to be sure there are no // changes debug_assert!(!self.persistence_state.in_progress()); - let mut blocks_to_persist = Vec::new(); - let mut current_hash = self.state.tree_state.canonical_block_hash(); - let last_persisted_number = self.persistence_state.last_persisted_block.number; + let prev_partial_state_trie = self.persistence_state.last_state_trie_persisted_block.number; + let prev_db_tip = self.persistence_state.last_persisted_block.number; let canonical_head_number = self.state.tree_state.canonical_block_number(); - let target_number = match target { - PersistTarget::Head => canonical_head_number, + let (new_db_tip, new_partial_state_trie) = match target { + PersistTarget::Head => (canonical_head_number, canonical_head_number), PersistTarget::Threshold => { - canonical_head_number.saturating_sub(self.config.memory_block_buffer_target()) + if self.building_payload || !self.backfill_sync_state.is_idle() { + return None + } + + if canonical_head_number.saturating_sub(prev_db_tip) <= + self.config.persistence_threshold() + { + return None + } + + let new_db_tip = + canonical_head_number.saturating_sub(self.config.memory_block_buffer_target()); + if new_db_tip <= prev_db_tip { + return None + } + + let new_partial_state_trie = new_db_tip + .saturating_sub(self.config.num_state_masking_blocks()) + .max(prev_partial_state_trie); + (new_db_tip, new_partial_state_trie) } }; + debug_assert!( + new_db_tip >= prev_db_tip, + "disk reorg must be resolved before saving blocks" + ); + debug_assert!( + new_partial_state_trie >= prev_partial_state_trie, + "disk reorg must be resolved before saving state/trie" + ); + + if new_db_tip == prev_db_tip && new_partial_state_trie == prev_partial_state_trie { + return None + } + + let mut blocks = Vec::new(); + let mut current_hash = self.state.tree_state.canonical_block_hash(); + debug!( target: "engine::tree", ?current_hash, - ?last_persisted_number, + ?prev_partial_state_trie, + ?prev_db_tip, ?canonical_head_number, - ?target_number, - "Returning canonical blocks to persist" + ?new_partial_state_trie, + ?new_db_tip, + target = ?target, + "Returning save input" ); while let Some(block) = self.state.tree_state.blocks_by_hash.get(¤t_hash) { - if block.recovered_block().number() <= last_persisted_number { + if block.recovered_block().number() <= prev_partial_state_trie { break; } - if block.recovered_block().number() <= target_number { - blocks_to_persist.push(block.clone()); + if block.recovered_block().number() <= new_db_tip { + blocks.push(block.clone()); } current_hash = block.recovered_block().parent_hash(); } // Reverse the order so that the oldest block comes first - blocks_to_persist.reverse(); - - Ok(blocks_to_persist) + blocks.reverse(); + + Some(SaveBlocksInput::new( + blocks, + prev_db_tip, + prev_partial_state_trie, + new_db_tip, + new_partial_state_trie, + )) } /// This clears the blocks from the in-memory tree state that have been persisted to the @@ -2187,6 +2197,8 @@ where /// /// Assumes that `finish` has been called on the `persistence_state` at least once fn on_new_persisted_block(&mut self) -> ProviderResult<()> { + let in_memory_persisted_block = self.persistence_state.last_state_trie_persisted_block; + // If we have an on-disk reorg, we need to handle it first before touching the in-memory // state. if let Some(remove_above) = self.find_disk_reorg()? { @@ -2195,11 +2207,11 @@ where } let finalized = self.state.forkchoice_state_tracker.last_valid_finalized(); - self.remove_before(self.persistence_state.last_persisted_block, finalized)?; - self.canonical_in_memory_state.remove_persisted_blocks(BlockNumHash { - number: self.persistence_state.last_persisted_block.number, - hash: self.persistence_state.last_persisted_block.hash, - }); + self.remove_before(in_memory_persisted_block, finalized)?; + self.canonical_in_memory_state.remove_persisted_blocks_until( + self.persistence_state.last_persisted_block, + in_memory_persisted_block.number, + ); self.state.set_pending_sparse_trie_prune(self.should_prune_sparse_trie()); Ok(()) } diff --git a/crates/engine/tree/src/tree/persistence_state.rs b/crates/engine/tree/src/tree/persistence_state.rs index c3ab00dbece..7dd78944ac5 100644 --- a/crates/engine/tree/src/tree/persistence_state.rs +++ b/crates/engine/tree/src/tree/persistence_state.rs @@ -22,7 +22,6 @@ use crate::persistence::PersistenceResult; use alloy_eips::BlockNumHash; -use alloy_primitives::B256; use crossbeam_channel::Receiver as CrossbeamReceiver; use reth_primitives_traits::FastInstant as Instant; use tracing::trace; @@ -30,10 +29,13 @@ use tracing::trace; /// The state of the persistence task. #[derive(Debug)] pub struct PersistenceState { - /// Hash and number of the last block persisted. + /// Hash and number of the highest block whose non-state/trie outputs are persisted. /// - /// This tracks the chain height that is persisted on disk + /// This tracks the highest canonical block with durable block/static-file/plain-state data. pub(crate) last_persisted_block: BlockNumHash, + /// Hash and number of the highest block whose state/trie outputs were processed for + /// persistence. + pub(crate) last_state_trie_persisted_block: BlockNumHash, /// Receiver end of channel where the result of the persistence task will be /// sent when done. A None value means there's no persistence task in progress. pub(crate) rx: @@ -76,13 +78,18 @@ impl PersistenceState { /// Sets state for a finished persistence task. pub(crate) fn finish( &mut self, - last_persisted_block_hash: B256, - last_persisted_block_number: u64, + last_persisted_block: BlockNumHash, + last_state_trie_persisted_block: BlockNumHash, ) { - trace!(target: "engine::tree", block= %last_persisted_block_number, hash=%last_persisted_block_hash, "updating persistence state"); + trace!( + target: "engine::tree", + last_persisted_block = %last_persisted_block.number, + last_state_trie_persisted_block = %last_state_trie_persisted_block.number, + "updating persistence state" + ); self.rx = None; - self.last_persisted_block = - BlockNumHash::new(last_persisted_block_number, last_persisted_block_hash); + self.last_persisted_block = last_persisted_block; + self.last_state_trie_persisted_block = last_state_trie_persisted_block; } } diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index af5580ddc7f..cec99a52726 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -243,7 +243,11 @@ impl TestHarness { engine_api_tree_state, canonical_in_memory_state, persistence_handle, - PersistenceState { last_persisted_block: BlockNumHash::default(), rx: None }, + PersistenceState { + last_persisted_block: BlockNumHash::default(), + last_state_trie_persisted_block: BlockNumHash::default(), + rx: None, + }, payload_builder, tree_config, EngineApiKind::Ethereum, @@ -597,10 +601,8 @@ async fn test_tree_persist_blocks() { fn on_new_persisted_block_queues_sparse_trie_prune_request() { let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); - test_harness - .tree - .persistence_state - .finish(blocks[0].recovered_block().hash(), blocks[0].recovered_block().number); + let persisted = blocks[0].recovered_block().num_hash(); + test_harness.tree.persistence_state.finish(persisted, persisted); test_harness.tree.on_new_persisted_block().unwrap(); @@ -611,10 +613,8 @@ fn on_new_persisted_block_queues_sparse_trie_prune_request() { fn on_new_persisted_block_queues_sparse_trie_prune_with_in_memory_blocks() { let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); - test_harness - .tree - .persistence_state - .finish(blocks[0].recovered_block().hash(), blocks[0].recovered_block().number); + let persisted = blocks[0].recovered_block().num_hash(); + test_harness.tree.persistence_state.finish(persisted, persisted); test_harness.tree.on_new_persisted_block().unwrap(); @@ -633,10 +633,8 @@ fn on_new_persisted_block_skips_sparse_trie_prune_when_state_root_task_disabled( for config in configs { let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); test_harness.tree.config = config; - test_harness - .tree - .persistence_state - .finish(blocks[0].recovered_block().hash(), blocks[0].recovered_block().number); + let persisted = blocks[0].recovered_block().num_hash(); + test_harness.tree.persistence_state.finish(persisted, persisted); test_harness.tree.on_new_persisted_block().unwrap(); @@ -897,7 +895,8 @@ fn test_backpressure_waits_for_persistence_before_reading_incoming() { std::thread::sleep(Duration::from_millis(10)); persist_tx .send(PersistenceResult { - last_block: Some(persisted), + last_block: persisted, + last_state_trie_block: persisted, commit_duration: Some(Duration::ZERO), }) .unwrap(); @@ -1015,7 +1014,8 @@ async fn test_tree_state_on_new_head_reorg() { // send the response so we can advance again sender .send(PersistenceResult { - last_block: Some(blocks[1].recovered_block().num_hash()), + last_block: blocks[1].recovered_block().num_hash(), + last_state_trie_block: blocks[1].recovered_block().num_hash(), commit_duration: Some(Duration::ZERO), }) .unwrap(); @@ -1153,6 +1153,8 @@ async fn test_get_canonical_blocks_to_persist() { let last_persisted_block_number = 3; test_harness.tree.persistence_state.last_persisted_block = blocks[last_persisted_block_number as usize].recovered_block.num_hash(); + test_harness.tree.persistence_state.last_state_trie_persisted_block = + blocks[last_persisted_block_number as usize].recovered_block.num_hash(); let persistence_threshold = 4; let memory_block_buffer_target = 3; @@ -1160,8 +1162,8 @@ async fn test_get_canonical_blocks_to_persist() { .with_persistence_threshold(persistence_threshold) .with_memory_block_buffer_target(memory_block_buffer_target); - let blocks_to_persist = - test_harness.tree.get_canonical_blocks_to_persist(PersistTarget::Threshold).unwrap(); + let input = test_harness.tree.get_save_blocks_input(PersistTarget::Threshold).unwrap(); + let blocks_to_persist = input.persist_rest_blocks(); let expected_blocks_to_persist_length: usize = (canonical_head_number - memory_block_buffer_target - last_persisted_block_number) @@ -1180,8 +1182,8 @@ async fn test_get_canonical_blocks_to_persist() { assert!(test_harness.tree.state.tree_state.sealed_header_by_hash(&fork_block_hash).is_some()); - let blocks_to_persist = - test_harness.tree.get_canonical_blocks_to_persist(PersistTarget::Threshold).unwrap(); + let input = test_harness.tree.get_save_blocks_input(PersistTarget::Threshold).unwrap(); + let blocks_to_persist = input.persist_rest_blocks(); assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length); // check that the fork block is not included in the blocks to persist @@ -1201,6 +1203,54 @@ async fn test_get_canonical_blocks_to_persist() { ); } +#[test] +fn test_threshold_persistence_with_state_masking_blocks() { + let mut test_harness = TestHarness::new(MAINNET.clone()); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(0..9).collect(); + test_harness = test_harness.with_blocks(blocks.clone()); + test_harness.tree.persistence_state.last_state_trie_persisted_block = + blocks[1].recovered_block().num_hash(); + test_harness.tree.persistence_state.last_persisted_block = + blocks[3].recovered_block().num_hash(); + test_harness.tree.config = TreeConfig::default() + .with_persistence_threshold(4) + .with_memory_block_buffer_target(1) + .with_num_state_masking_blocks(2); + + test_harness.tree.advance_persistence().unwrap(); + let action = test_harness.action_rx.recv().unwrap(); + let PersistenceAction::SaveBlocks(input, sender) = action else { + panic!("expected save blocks action, got {action:?}") + }; + + let persisted_tip = blocks[7].recovered_block().num_hash(); + let state_trie_tip = blocks[5].recovered_block().num_hash(); + assert_eq!(input.new_db_tip(), persisted_tip.number); + assert_eq!(input.new_partial_state_trie(), state_trie_tip.number); + sender + .send(PersistenceResult { + last_block: persisted_tip, + last_state_trie_block: state_trie_tip, + commit_duration: Some(Duration::ZERO), + }) + .unwrap(); + test_harness.tree.try_poll_persistence().unwrap(); + + assert_eq!(test_harness.tree.persistence_state.last_persisted_block, persisted_tip); + assert_eq!(test_harness.tree.persistence_state.last_state_trie_persisted_block, state_trie_tip); + assert_eq!( + test_harness.tree.canonical_in_memory_state.get_persisted_num_hash(), + Some(persisted_tip) + ); + + let removed_hash = blocks[state_trie_tip.number as usize].recovered_block().hash(); + let retained_hash = blocks[(state_trie_tip.number + 1) as usize].recovered_block().hash(); + assert!(test_harness.tree.state.tree_state.executed_block_by_hash(removed_hash).is_none()); + assert!(test_harness.tree.state.tree_state.executed_block_by_hash(retained_hash).is_some()); + assert!(test_harness.tree.canonical_in_memory_state.state_by_hash(removed_hash).is_none()); + assert!(test_harness.tree.canonical_in_memory_state.state_by_hash(retained_hash).is_some()); +} + #[tokio::test] async fn test_engine_tree_fcu_missing_head() { let chain_spec = MAINNET.clone(); @@ -2295,7 +2345,8 @@ mod forkchoice_updated_tests { last_persisted_number = last.number; sender .send(PersistenceResult { - last_block: Some(last), + last_block: last, + last_state_trie_block: last, commit_duration: Some(Duration::ZERO), }) .unwrap(); @@ -2305,6 +2356,45 @@ mod forkchoice_updated_tests { // Ensure we persisted right to the tip assert_eq!(last_persisted_number, canonical_tip); } + + #[test] + fn test_engine_termination_catches_up_state_trie_at_database_tip() { + let chain_spec = MAINNET.clone(); + let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..11).collect(); + let database_tip = blocks.last().unwrap().recovered_block().num_hash(); + let state_trie_tip = blocks[4].recovered_block().num_hash(); + let mut test_harness = TestHarness::new(chain_spec).with_blocks(blocks); + test_harness.tree.persistence_state.last_persisted_block = database_tip; + test_harness.tree.persistence_state.last_state_trie_persisted_block = state_trie_tip; + + let (terminate_tx, terminate_rx) = oneshot::channel(); + let to_tree_tx = test_harness.to_tree_tx.clone(); + let action_rx = test_harness.action_rx; + spawn_os_thread("engine", || test_harness.tree.run()); + + to_tree_tx + .send(FromEngine::Event(FromOrchestrator::Terminate { tx: terminate_tx })) + .unwrap(); + + let action = action_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + let PersistenceAction::SaveBlocks(input, sender) = action else { + panic!("expected state/trie catch-up save, got {action:?}") + }; + assert_eq!(input.new_db_tip(), database_tip.number); + assert_eq!(input.new_partial_state_trie(), database_tip.number); + assert!(input.persist_rest_blocks().is_empty()); + + sender + .send(PersistenceResult { + last_block: database_tip, + last_state_trie_block: database_tip, + commit_duration: Some(Duration::ZERO), + }) + .unwrap(); + + terminate_rx.blocking_recv().unwrap(); + } } /// Tests that `on_valid_downloaded_block` triggers a download for the actual head block when diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index 2d16bf1277c..bfd54c40669 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -8,7 +8,7 @@ use eyre::ensure; use reth_cli_util::{parse_duration_from_secs_or_ms, parsers::format_duration_as_secs_or_ms}; use reth_engine_primitives::{ TreeConfig, DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, - DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, + DEFAULT_NUM_STATE_MASKING_BLOCKS, DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, }; use std::{sync::OnceLock, time::Duration}; @@ -27,6 +27,7 @@ static ENGINE_DEFAULTS: OnceLock = OnceLock::new(); pub struct DefaultEngineValues { persistence_threshold: u64, persistence_backpressure_threshold: u64, + num_state_masking_blocks: u64, memory_block_buffer_target: u64, invalid_header_hit_eviction_threshold: u8, state_cache_disabled: bool, @@ -80,6 +81,12 @@ impl DefaultEngineValues { self } + /// Set the default number of state masking blocks. + pub const fn with_num_state_masking_blocks(mut self, v: u64) -> Self { + self.num_state_masking_blocks = v; + self + } + /// Set the default memory block buffer target pub const fn with_memory_block_buffer_target(mut self, v: u64) -> Self { self.memory_block_buffer_target = v; @@ -257,6 +264,7 @@ impl Default for DefaultEngineValues { Self { persistence_threshold: DEFAULT_PERSISTENCE_THRESHOLD, persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD, + num_state_masking_blocks: DEFAULT_NUM_STATE_MASKING_BLOCKS, memory_block_buffer_target: DEFAULT_MEMORY_BLOCK_BUFFER_TARGET, invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD, state_cache_disabled: false, @@ -318,6 +326,21 @@ pub struct EngineArgs { #[arg(long = "engine.persistence-backpressure-threshold")] pub persistence_backpressure_threshold: Option, + /// Configure how many of the blocks being persisted should only mask state/trie writes instead + /// of durably persisting their state/trie updates in the current cycle. + #[cfg_attr( + feature = "partial-persistence", + arg( + long = "engine.num-state-masking-blocks", + default_value_t = DefaultEngineValues::get_global().num_state_masking_blocks + ) + )] + #[cfg_attr( + not(feature = "partial-persistence"), + arg(skip = DefaultEngineValues::get_global().num_state_masking_blocks) + )] + pub num_state_masking_blocks: u64, + /// Configure the target number of blocks to keep in memory. /// /// If omitted, this defaults to the lesser of `--engine.persistence-threshold` and the @@ -563,6 +586,7 @@ impl Default for EngineArgs { let DefaultEngineValues { persistence_threshold, persistence_backpressure_threshold: _, + num_state_masking_blocks, memory_block_buffer_target: _, invalid_header_hit_eviction_threshold, state_cache_disabled, @@ -595,6 +619,7 @@ impl Default for EngineArgs { Self { persistence_threshold, persistence_backpressure_threshold: None, + num_state_masking_blocks, memory_block_buffer_target: None, invalid_header_hit_eviction_threshold, state_root_task_compare_updates, @@ -669,6 +694,17 @@ impl EngineArgs { memory_block_buffer_target, self.persistence_threshold, ); + ensure!( + self.num_state_masking_blocks == 0 || + matches!( + self.num_state_masking_blocks.checked_add(memory_block_buffer_target), + Some(window) if window < self.persistence_threshold + ), + "--engine.num-state-masking-blocks ({}) + --engine.memory-block-buffer-target ({}) must be less than --engine.persistence-threshold ({})", + self.num_state_masking_blocks, + memory_block_buffer_target, + self.persistence_threshold, + ); ensure!( !self.state_cache_disabled || !self.txpool_prewarming_enabled, "--engine.txpool-prewarming conflicts with --engine.disable-state-cache" @@ -690,6 +726,7 @@ impl EngineArgs { .with_persistence_backpressure_threshold(self.persistence_backpressure_threshold()) .with_persistence_threshold(self.persistence_threshold) .with_memory_block_buffer_target(self.memory_block_buffer_target()) + .with_num_state_masking_blocks(self.num_state_masking_blocks) .with_invalid_header_hit_eviction_threshold(self.invalid_header_hit_eviction_threshold) .without_state_cache(self.state_cache_disabled) .without_prewarming(self.prewarming_disabled) @@ -843,6 +880,7 @@ mod tests { let args = EngineArgs { persistence_threshold: 100, persistence_backpressure_threshold: Some(101), + num_state_masking_blocks: DEFAULT_NUM_STATE_MASKING_BLOCKS, memory_block_buffer_target: Some(50), invalid_header_hit_eviction_threshold: 7, legacy_state_root_task_enabled: true, @@ -956,6 +994,57 @@ mod tests { assert!(err.contains("engine.persistence-threshold")); } + #[cfg(feature = "partial-persistence")] + #[test] + fn test_parse_num_state_masking_blocks() { + let args = CommandParser::::parse_from([ + "reth", + "--engine.persistence-threshold", + "13", + "--engine.num-state-masking-blocks", + "7", + ]) + .args; + + assert_eq!(args.tree_config().num_state_masking_blocks(), 7); + } + + #[cfg(not(feature = "partial-persistence"))] + #[test] + fn num_state_masking_blocks_is_hidden_without_partial_persistence() { + assert!(CommandParser::::try_parse_from([ + "reth", + "--engine.num-state-masking-blocks", + "1", + ]) + .is_err()); + } + + #[test] + fn validate_rejects_state_masking_window_at_or_above_threshold() { + let args = EngineArgs { + persistence_threshold: 4, + num_state_masking_blocks: 2, + memory_block_buffer_target: Some(2), + ..EngineArgs::default() + }; + + let err = args.validate().unwrap_err().to_string(); + assert!(err.contains("engine.num-state-masking-blocks")); + } + + #[test] + fn validate_rejects_overflowing_state_masking_window() { + let args = EngineArgs { + persistence_threshold: 7, + num_state_masking_blocks: u64::MAX, + ..EngineArgs::default() + }; + + let err = args.validate().unwrap_err().to_string(); + assert!(err.contains("engine.num-state-masking-blocks")); + } + #[test] fn default_memory_block_buffer_target_is_bounded_by_persistence_threshold() { let args = CommandParser::::parse_from([ diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index e8133923e35..2accb69ddf1 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -15,9 +15,9 @@ use crate::{ BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, DBProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, HeaderProvider, HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, - LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, ProviderError, - PruneCheckpointReader, PruneCheckpointWriter, RawRocksDBBatch, RevertsInit, RocksBatchArg, - RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateWriter, + LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, PersistenceFrontiers, + ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RawRocksDBBatch, RevertsInit, + RocksBatchArg, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, TransactionsProvider, TransactionsProviderExt, TrieWriter, }; @@ -2393,6 +2393,38 @@ impl StageCheckpointWriter for DatabaseProvider DatabaseProvider { + /// Updates pipeline checkpoints after an unwind while preserving an explicitly lagging + /// state/trie frontier. + fn update_pipeline_stages_after_unwind( + &self, + block_number: BlockNumber, + ) -> ProviderResult { + let partial_state_trie = self + .get_stage_checkpoint(StageId::Finish)? + .map(|checkpoint| { + checkpoint + .finish_stage_checkpoint() + .and_then(|finish| finish.partial_state_trie()) + .unwrap_or(checkpoint.block_number) + }) + .unwrap_or(block_number) + .min(block_number); + + self.update_pipeline_stages(block_number, true)?; + if partial_state_trie < block_number { + self.save_stage_checkpoint( + StageId::Finish, + StageCheckpoint::new(block_number).with_finish_stage_checkpoint(FinishCheckpoint { + partial_state_trie: Some(partial_state_trie), + }), + )?; + } + + Ok(PersistenceFrontiers { db_tip: block_number, partial_state_trie }) + } +} + impl StorageReader for DatabaseProvider { fn plain_state_storages( &self, @@ -3561,12 +3593,15 @@ impl BlockExecutionWriter self.remove_blocks_above(block)?; // Update pipeline progress - self.update_pipeline_stages(block, true)?; + self.update_pipeline_stages_after_unwind(block)?; Ok(Chain::new(blocks, execution_state, BTreeMap::new())) } - fn remove_block_and_execution_above(&self, block: BlockNumber) -> ProviderResult<()> { + fn remove_block_and_execution_above( + &self, + block: BlockNumber, + ) -> ProviderResult { self.unwind_trie_state_from(block + 1)?; // remove execution res @@ -3577,9 +3612,7 @@ impl BlockExecutionWriter self.remove_blocks_above(block)?; // Update pipeline progress - self.update_pipeline_stages(block, true)?; - - Ok(()) + self.update_pipeline_stages_after_unwind(block) } } @@ -4850,6 +4883,37 @@ mod tests { ); } + #[cfg(feature = "partial-persistence")] + #[test] + fn remove_block_and_execution_above_returns_persistence_frontiers() { + let factory = create_test_provider_factory(); + let mut test_block_builder = TestBlockBuilder::eth().with_state(); + + let genesis = test_block_builder.get_executed_blocks(0..1).next().unwrap(); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..5).collect(); + + let provider_rw = factory.provider_rw().unwrap(); + save_genesis(&provider_rw, &genesis).unwrap(); + provider_rw.commit().unwrap(); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new(blocks, 0, 0, 4, 2); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let provider_rw = factory.provider_rw().unwrap(); + let frontiers = provider_rw.remove_block_and_execution_above(3).unwrap(); + assert_eq!(frontiers, PersistenceFrontiers { db_tip: 3, partial_state_trie: 2 }); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let checkpoint = provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); + assert_eq!( + checkpoint.finish_stage_checkpoint().and_then(|finish| finish.partial_state_trie()), + Some(2) + ); + } + #[test] fn test_prunable_receipts_logic() { let insert_blocks = diff --git a/crates/storage/storage-api/src/block_writer.rs b/crates/storage/storage-api/src/block_writer.rs index 124e29110de..bf4adae93ad 100644 --- a/crates/storage/storage-api/src/block_writer.rs +++ b/crates/storage/storage-api/src/block_writer.rs @@ -22,7 +22,12 @@ pub trait BlockExecutionWriter: /// Remove all of the blocks above the provided number and their execution result /// /// The passed block number will stay in the database. - fn remove_block_and_execution_above(&self, block: BlockNumber) -> ProviderResult<()>; + /// + /// Returns the persistence frontiers after removal. + fn remove_block_and_execution_above( + &self, + block: BlockNumber, + ) -> ProviderResult; } impl BlockExecutionWriter for &T { @@ -33,11 +38,25 @@ impl BlockExecutionWriter for &T { (*self).take_block_and_execution_above(block) } - fn remove_block_and_execution_above(&self, block: BlockNumber) -> ProviderResult<()> { + fn remove_block_and_execution_above( + &self, + block: BlockNumber, + ) -> ProviderResult { (*self).remove_block_and_execution_above(block) } } +/// The database and state/trie persistence frontiers. +/// +/// The state/trie frontier is never ahead of the database frontier. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct PersistenceFrontiers { + /// The highest block whose non-state/trie outputs are persisted. + pub db_tip: BlockNumber, + /// The highest block whose state/trie outputs are persisted. + pub partial_state_trie: BlockNumber, +} + /// Block Writer #[auto_impl::auto_impl(&, Arc, Box)] pub trait BlockWriter { From e2563b7a4d621ef7f4e7b1e5a7edc34e1e768f6f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Fri, 31 Jul 2026 12:18:00 +0200 Subject: [PATCH 072/148] feat: add -`-with-receipts` to `import-era` to store receipts (#26436) Co-authored-by: Matthias Seitz --- Cargo.lock | 3 + crates/cli/commands/src/common.rs | 38 +- crates/cli/commands/src/import_era.rs | 75 +- crates/era-utils/Cargo.toml | 3 + crates/era-utils/src/history.rs | 1402 ++++++++++++++++- crates/era-utils/src/lib.rs | 4 +- crates/era-utils/tests/it/history.rs | 98 +- crates/era/src/ere/types/execution.rs | 12 +- crates/stages/stages/src/stages/era.rs | 8 +- crates/storage/provider/src/test_utils/mod.rs | 35 +- docs/vocs/docs/pages/cli/reth/import-era.mdx | 8 + 11 files changed, 1594 insertions(+), 92 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 95fba14dc24..971d49f1190 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8544,6 +8544,7 @@ dependencies = [ "eyre", "futures-util", "reqwest", + "reth-codecs", "reth-db-api", "reth-db-common", "reth-era", @@ -8553,7 +8554,9 @@ dependencies = [ "reth-fs-util", "reth-primitives-traits", "reth-provider", + "reth-prune-types", "reth-stages-types", + "reth-static-file-types", "reth-storage-api", "tempfile", "tokio", diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index fd7b5aacb35..2748ba5ec9c 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -129,7 +129,7 @@ impl EnvironmentArgs { info!(target: "reth::cli", ?db_path, ?sf_path, "Opening storage"); let genesis_block_number = self.chain.genesis().number.unwrap_or_default(); let (db, sfp) = match access { - AccessRights::RW => ( + AccessRights::RW | AccessRights::RwInconsistent => ( init_db(db_path, self.db.database_args())?, StaticFileProviderBuilder::read_write(sf_path) .with_metrics() @@ -179,11 +179,11 @@ impl EnvironmentArgs { Ok(Environment { config, provider_factory, data_dir }) } - /// Returns a [`ProviderFactory`] after executing consistency checks. + /// Returns a [`ProviderFactory`] after executing consistency checks unless `access` permits + /// inconsistent storage. /// - /// If it's a read-write environment and an issue is found, it will attempt to heal (including a - /// pipeline unwind). Otherwise, it will print out a warning, advising the user to restart the - /// node to heal. + /// Checked read-write access heals inconsistencies (including a pipeline unwind), while checked + /// read-only access warns that the node must be restarted to heal. fn create_provider_factory( &self, config: &Config, @@ -213,7 +213,7 @@ impl EnvironmentArgs { .with_bal_store(bal_store); // Check for consistency between database and static files. - if !access.is_read_only_inconsistent() && + if !access.skips_consistency_check() && let Some(unwind_target) = factory.static_file_provider().check_consistency(&factory.provider()?)? { @@ -277,6 +277,8 @@ pub struct Environment { pub enum AccessRights { /// Read-write access RW, + /// Read-write access with possibly inconsistent data + RwInconsistent, /// Read-only access RO, /// Read-only access with possibly inconsistent data @@ -286,7 +288,7 @@ pub enum AccessRights { impl AccessRights { /// Returns `true` if it requires read-write access to the environment. pub const fn is_read_write(&self) -> bool { - matches!(self, Self::RW) + matches!(self, Self::RW | Self::RwInconsistent) } /// Returns `true` if it requires read-only access to the environment with possibly inconsistent @@ -294,6 +296,11 @@ impl AccessRights { pub const fn is_read_only_inconsistent(&self) -> bool { matches!(self, Self::RoInconsistent) } + + /// Returns `true` if storage consistency checks should be skipped. + pub const fn skips_consistency_check(&self) -> bool { + matches!(self, Self::RwInconsistent | Self::RoInconsistent) + } } /// Helper alias to satisfy `FullNodeTypes` bound on [`Node`] trait generic. @@ -359,3 +366,20 @@ where { type Components = Comp; } + +#[cfg(test)] +mod tests { + use super::AccessRights; + + #[test] + fn inconsistent_access_rights_skip_consistency_checks() { + assert!(AccessRights::RwInconsistent.is_read_write()); + assert!(AccessRights::RwInconsistent.skips_consistency_check()); + assert!(!AccessRights::RW.skips_consistency_check()); + + assert!(!AccessRights::RoInconsistent.is_read_write()); + assert!(AccessRights::RoInconsistent.is_read_only_inconsistent()); + assert!(AccessRights::RoInconsistent.skips_consistency_check()); + assert!(!AccessRights::RO.skips_consistency_check()); + } +} diff --git a/crates/cli/commands/src/import_era.rs b/crates/cli/commands/src/import_era.rs index 56e4e760502..c4d35bb7a9e 100644 --- a/crates/cli/commands/src/import_era.rs +++ b/crates/cli/commands/src/import_era.rs @@ -32,6 +32,15 @@ pub struct ImportEraCommand { /// import ends. By default all available blocks are imported. #[arg(long, value_name = "TO_BLOCK", verbatim_doc_comment)] to_block: Option, + + /// Backfill the `Receipts` static file segment from its tip up to the Execution checkpoint. + /// + /// Only a missing tail is detected and repaired, never a gap inside the existing segment. The + /// backfill must reach the checkpoint exactly, so this cannot bootstrap a fresh database: + /// above the checkpoint receipts are pruned on the next node start, below it the node cannot + /// start. Byzantium onwards only, from `.era1` or `.ere` files that carry receipts. + #[arg(long, verbatim_doc_comment)] + with_receipts: bool, } #[derive(Debug, Args)] @@ -78,16 +87,38 @@ impl> ImportEraC { info!(target: "reth::cli", "reth {} starting", version_metadata().short_version); - let Environment { provider_factory, config, .. } = - self.env.init::(AccessRights::RW, runtime)?; + // Receipt backfill expects Receipts to trail Execution, so skip the normal consistency + // check that would unwind execution before the repair. + let access = + if self.with_receipts { AccessRights::RwInconsistent } else { AccessRights::RW }; + let Environment { provider_factory, config, .. } = self.env.init::(access, runtime)?; let mut hash_collector = Collector::new(config.stages.etl.file_size, config.stages.etl.dir); - let next_block = provider_factory - .static_file_provider() + let static_file_provider = provider_factory.static_file_provider(); + // The chain's first block, which is not necessarily 0: reth supports a non-zero genesis. + let genesis_block_number = static_file_provider.genesis_block_number(); + let headers_tip = static_file_provider .get_highest_static_file_block(StaticFileSegment::Headers) - .unwrap_or_default() + - 1; + .unwrap_or(genesis_block_number); + + // With `--with-receipts`, resume from the receipts tip so files covering already-imported + // headers are re-read to backfill their receipts. + let resume_block = if self.with_receipts { + let receipts_tip = static_file_provider + .get_highest_static_file_block(StaticFileSegment::Receipts) + .unwrap_or(genesis_block_number); + headers_tip.min(receipts_tip) + } else { + headers_tip + }; + let next_block = resume_block + 1; + + // Pre-Byzantium receipts commit to a post-state root, so their receipts root can't be + // recomputed from what the node stores. The logs bloom is checked on every fork. + let chain_spec = self.env.chain.clone(); + let is_receipt_verifiable = + move |number: u64| chain_spec.is_byzantium_active_at_block(number); if let Some(path) = self.import.path { let era_type = EraFileType::from_dir(&path)?.ok_or_else(|| { @@ -97,7 +128,9 @@ impl> ImportEraC ) })?; - info!(target: "reth::cli", ?era_type, path = %path.display(), to_block = ?self.to_block, "Starting ERA import"); + info!(target: "reth::cli", ?era_type, path = %path.display(), to_block = ?self.to_block, with_receipts = self.with_receipts, "Starting ERA import"); + + check_receipts_supported(self.with_receipts, era_type)?; match era_type { EraFileType::Era => era::import::( @@ -105,18 +138,24 @@ impl> ImportEraC &provider_factory, &mut hash_collector, self.to_block, + self.with_receipts, + &is_receipt_verifiable, )?, EraFileType::Ere => era::import::( read_dir(path, next_block)?, &provider_factory, &mut hash_collector, self.to_block, + self.with_receipts, + &is_receipt_verifiable, )?, EraFileType::Era1 => era::import::( read_dir(path, next_block)?, &provider_factory, &mut hash_collector, self.to_block, + self.with_receipts, + &is_receipt_verifiable, )?, }; } else { @@ -126,7 +165,9 @@ impl> ImportEraC }; let era_type = EraFileType::from_url(url.as_str()); - info!(target: "reth::cli", ?era_type, %url, to_block = ?self.to_block, "Starting ERA import"); + info!(target: "reth::cli", ?era_type, %url, to_block = ?self.to_block, with_receipts = self.with_receipts, "Starting ERA import"); + + check_receipts_supported(self.with_receipts, era_type)?; let folder = self.env.datadir.resolve_datadir(self.env.chain.chain()).data_dir().join("era"); @@ -149,18 +190,24 @@ impl> ImportEraC &provider_factory, &mut hash_collector, self.to_block, + self.with_receipts, + &is_receipt_verifiable, )?, EraFileType::Era1 => era::import::( stream, &provider_factory, &mut hash_collector, self.to_block, + self.with_receipts, + &is_receipt_verifiable, )?, EraFileType::Era => era::import::( stream, &provider_factory, &mut hash_collector, self.to_block, + self.with_receipts, + &is_receipt_verifiable, )?, }; } @@ -169,6 +216,18 @@ impl> ImportEraC } } +/// Errors if `--with-receipts` was passed for the consensus `.era` format, which carries no +/// receipt data at all. +fn check_receipts_supported(with_receipts: bool, era_type: EraFileType) -> eyre::Result<()> { + if with_receipts && era_type == EraFileType::Era { + return Err(eyre!( + "--with-receipts is not supported for `.era` files: they contain no receipt data. \ + Use `.era1` or `.ere` files instead." + )); + } + Ok(()) +} + impl ImportEraCommand { /// Returns the underlying chain being used to run this command pub fn chain_spec(&self) -> Option<&Arc> { diff --git a/crates/era-utils/Cargo.toml b/crates/era-utils/Cargo.toml index 355f8142b57..821dbb48c38 100644 --- a/crates/era-utils/Cargo.toml +++ b/crates/era-utils/Cargo.toml @@ -16,6 +16,7 @@ alloy-primitives.workspace = true alloy-rlp.workspace = true # reth +reth-codecs.workspace = true reth-db-api.workspace = true reth-era.workspace = true reth-era-downloader.workspace = true @@ -40,6 +41,8 @@ reth-provider.workspace = true reth-provider.features = ["test-utils"] reth-db-common.workspace = true reth-ethereum-primitives.workspace = true +reth-prune-types.workspace = true +reth-static-file-types.workspace = true # async tokio-util.workspace = true diff --git a/crates/era-utils/src/history.rs b/crates/era-utils/src/history.rs index 09e200a0e07..364335a120e 100644 --- a/crates/era-utils/src/history.rs +++ b/crates/era-utils/src/history.rs @@ -1,6 +1,11 @@ -use alloy_consensus::BlockHeader; -use alloy_primitives::{BlockHash, BlockNumber, U256}; +use alloy_consensus::{ + proofs::calculate_receipt_root, BlockHeader, Eip658Value, ReceiptEnvelope, ReceiptWithBloom, + RlpDecodableReceipt, TxReceipt, +}; +use alloy_primitives::{BlockHash, BlockNumber, Bloom, U256}; +use alloy_rlp::Decodable; use futures_util::{Stream, StreamExt}; +use reth_codecs::Compact; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW}, table::Value, @@ -18,58 +23,77 @@ use reth_era::{ use reth_era_downloader::EraMeta; use reth_etl::Collector; use reth_fs_util as fs; -use reth_primitives_traits::{Block, BlockBody, FullBlockBody, FullBlockHeader, NodePrimitives}; +use reth_primitives_traits::{ + Block, BlockBody, FullBlockBody, FullBlockHeader, NodePrimitives, Receipt, +}; use reth_provider::{ - providers::StaticFileProviderRWRefMut, BlockReader, BlockWriter, StaticFileProviderFactory, - StaticFileSegment, StaticFileWriter, + providers::StaticFileProviderRWRefMut, BlockReader, BlockWriter, EitherWriter, + EitherWriterDestination, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, }; use reth_stages_types::{ CheckpointBlockRange, EntitiesCheckpoint, HeadersCheckpoint, StageCheckpoint, StageId, }; use reth_storage_api::{ - errors::ProviderResult, DBProvider, DatabaseProviderFactory, NodePrimitivesProvider, - StageCheckpointWriter, + errors::{ProviderError, ProviderResult}, + BlockBodyIndicesProvider, BlockHashReader, DBProvider, DatabaseProviderFactory, + NodePrimitivesProvider, StageCheckpointReader, StageCheckpointWriter, StorageSettingsCache, }; use std::{collections::Bound, error::Error, ops::RangeBounds, sync::mpsc}; use tracing::info; -/// Reads execution `(header, body)` pairs out of an ERA file. +/// A decoded ERA block: header, body, and optionally its receipts. +type EraBlock = (BH, BB, Option>); + +/// The receipt type of the node primitives behind provider `P`. +type ReceiptOf

= <

::Primitives as NodePrimitives>::Receipt; + +/// Reads execution `(header, body, receipts)` tuples out of an ERA file. +/// +/// `receipts` is `None` when `decode_receipts` is `false`, or the file has none (`.era` never +/// does; `.ere` receipts are optional). `decode_receipts = false` skips decoding entirely. /// /// Per-format seam of the import pipeline. -pub trait EraBlockReader { +pub trait EraBlockReader { /// Opens the ERA file at `meta` and iterates its execution blocks. fn blocks( meta: &M, - ) -> eyre::Result>>; + decode_receipts: bool, + ) -> eyre::Result>>>; } /// [`EraBlockReader`] for `.era1` files. #[derive(Debug)] pub struct Era1; -impl EraBlockReader for Era1 +impl EraBlockReader for Era1 where BH: FullBlockHeader + Value, BB: FullBlockBody, + R: RlpDecodableReceipt, { fn blocks( meta: &M, - ) -> eyre::Result>> { + decode_receipts: bool, + ) -> eyre::Result>>> { let reader: Era1Reader = open(meta)?; - Ok(reader.iter().map(decode::)) + Ok(reader + .iter() + .map(move |block| decode_with_receipts::(block, decode_receipts))) } } -impl EraBlockReader for Ere +impl EraBlockReader for Ere where BH: FullBlockHeader + Value, BB: FullBlockBody, + R: RlpDecodableReceipt, { fn blocks( meta: &M, - ) -> eyre::Result>> { + decode_receipts: bool, + ) -> eyre::Result>>> { let reader: EreReader = open(meta)?; - Ok(reader.iter().map(Self::decode)) + Ok(reader.iter().map(move |block| Self::decode_with_receipts(block, decode_receipts))) } } @@ -78,8 +102,9 @@ where pub struct Ere; impl Ere { - /// Extracts a pair of [`FullBlockHeader`] and [`FullBlockBody`] from an ERE block tuple, whose - /// header and body are RLP-compressed. + /// Extracts a `(header, body)` pair from an ERE block tuple, whose header and body are + /// RLP-compressed. Ignores any receipts entry; callers that need receipts should use + /// [`decode_with_receipts`](Self::decode_with_receipts). pub fn decode(block: Result) -> eyre::Result<(BH, BB)> where BH: FullBlockHeader + Value, @@ -91,6 +116,33 @@ impl Ere { let body: BB = block.body.decode()?; Ok((header, body)) } + + /// Like [`decode`](Self::decode), but also extracts receipts when `decode_receipts` is `true`. + /// `receipts` is `None` if `decode_receipts` is `false`, or if the block tuple carries no + /// receipts entry (`.ere` receipts are optional per spec). + pub fn decode_with_receipts( + block: Result, + decode_receipts: bool, + ) -> eyre::Result> + where + BH: FullBlockHeader + Value, + BB: FullBlockBody, + R: RlpDecodableReceipt, + E: From + Error + Send + Sync + 'static, + { + let block = block?; + let header: BH = block.header.decode()?; + let body: BB = block.body.decode()?; + let number = header.number(); + let receipts = decode_receipts + .then(|| block.receipts.as_ref().map(|r| r.decode_receipts())) + .flatten() + .transpose()? + .map(|slim| receipts_from_envelopes(number, slim.into_iter().map(Into::into).collect())) + .transpose()?; + + Ok((header, body, receipts)) + } } /// [`EraBlockReader`] for consensus-layer `.era` files. @@ -98,21 +150,28 @@ impl Ere { /// `.era` files store consensus `SignedBeaconBlock`s. Post-merge /// blocks embed an execution payload; this source SSZ-decodes each beacon block, extracts that /// payload, and converts it into an execution `(header, body)` pair. Pre-merge slots carry no -/// payload and are skipped. +/// payload and are skipped. `.era` files carry no receipt data at all, so this source always +/// yields `None` for receipts. #[derive(Debug)] pub struct Era; -impl EraBlockReader for Era +impl EraBlockReader for Era where BH: FullBlockHeader, BB: FullBlockBody, { + /// `.era` files carry no receipt data, so `decode_receipts` has no effect. fn blocks( meta: &M, - ) -> eyre::Result>> { + _decode_receipts: bool, + ) -> eyre::Result>>> { let reader: EraReader = open(meta)?; let mut buf = Vec::new(); - Ok(reader.iter().filter_map(move |block| Self::decode(block, &mut buf).transpose())) + Ok(reader.iter().filter_map(move |block| { + Self::decode(block, &mut buf) + .map(|opt| opt.map(|(header, body)| (header, body, None))) + .transpose() + })) } } @@ -167,15 +226,25 @@ where /// When `to_block` is set, the import stops after reaching that block height; otherwise it /// continues until the source has no more files. /// +/// `store_receipts` backfills the `Receipts` segment from its tip up to the `Execution` +/// checkpoint, which it must reach exactly: receipts above the checkpoint are pruned on the next +/// node start, and receipts below it leave the node unable to start. Only a missing tail is +/// detected, never a gap inside the existing segment. +/// +/// `is_receipt_verifiable` decides which blocks have their receipts checked against the header, +/// as pre-Byzantium receipts can't be recomputed here. +/// /// Returns current block height. pub fn import( mut downloader: Downloader, provider_factory: &PF, hash_collector: &mut Collector, to_block: Option, + store_receipts: bool, + is_receipt_verifiable: &dyn Fn(BlockNumber) -> bool, ) -> eyre::Result where - S: EraBlockReader, + S: EraBlockReader::ProviderRW>>, B: Block

, BH: FullBlockHeader + Value, BB: FullBlockBody< @@ -187,9 +256,14 @@ where PF: DatabaseProviderFactory< ProviderRW: BlockWriter + DBProvider + + BlockBodyIndicesProvider + + BlockHashReader + StaticFileProviderFactory> - + StageCheckpointWriter, + + StageCheckpointReader + + StageCheckpointWriter + + StorageSettingsCache, > + StaticFileProviderFactory::ProviderRW as NodePrimitivesProvider>::Primitives>, + ReceiptOf<::ProviderRW>: Compact + Receipt, { let (tx, rx) = mpsc::channel(); @@ -203,28 +277,129 @@ where let static_file_provider = provider_factory.static_file_provider(); + // The chain's first block, which is not necessarily 0: reth supports a non-zero genesis. + let genesis_block_number = static_file_provider.genesis_block_number(); + // Consistency check of expected headers in static files vs DB is done on provider::sync_gap // when poll_execute_ready is polled. - let mut height = static_file_provider + let headers_tip = static_file_provider .get_highest_static_file_block(StaticFileSegment::Headers) - .unwrap_or_default(); + .unwrap_or(genesis_block_number); + + // When backfilling receipts, resume from the receipts tip so blocks whose headers were already + // imported still get their receipts; only blocks above `headers_tip` are written in full. + let receipts_tip = store_receipts + .then(|| static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts)); + let mut height = match receipts_tip { + Some(tip) => headers_tip.min(tip.unwrap_or(genesis_block_number)), + None => headers_tip, + }; + + // Only the executed range can hold receipts durably, so the repair has to land exactly on the + // Execution checkpoint. + let receipts_target = store_receipts + .then(|| -> eyre::Result<_> { + let provider = provider_factory.database_provider_rw()?; + + // Writing the segment under a config that routes receipts to the database would leave + // two sources and keep data the prune config wants dropped. + if !matches!( + EitherWriter::receipts_destination(&provider), + EitherWriterDestination::StaticFile + ) { + eyre::bail!( + "receipt import writes the Receipts static file segment, but this node's \ + prune configuration keeps receipts elsewhere. Remove the receipt pruning \ + configuration, or import without receipts" + ); + } + + let target = provider + .get_stage_checkpoint(StageId::Execution)? + .map(|checkpoint| checkpoint.block_number) + .unwrap_or(genesis_block_number); + + if target <= genesis_block_number { + eyre::bail!( + "receipt import repairs the receipt static files of an already-executed \ + range, but this database has executed no blocks. Sync the node first, or \ + import without receipts" + ); + } + if height >= target { + eyre::bail!( + "receipts already cover the executed range up to block {target}, nothing to \ + repair" + ); + } + if let Some(to_block) = to_block && + to_block != target + { + eyre::bail!( + "--to-block {to_block} does not match the Execution checkpoint {target}. A \ + receipt repair must cover the executed range exactly, so either drop \ + --to-block or set it to {target}" + ); + } + // Such a repair can never complete, so fail before committing the empty blocks that + // precede the first pre-Byzantium receipt. + if !is_receipt_verifiable(height + 1) { + eyre::bail!( + "receipt repair would start at block {}, which predates Byzantium. Those \ + receipts commit to a post-state root this node's receipt type cannot \ + represent, so the Receipts segment has to already cover the pre-Byzantium \ + range", + height + 1, + ); + } + + Ok(target) + }) + .transpose()?; + + // A segment that doesn't exist yet holds no receipts for genesis, which has none of its own. + // The backfill resumes at the block after it, so seed that entry the way `init_genesis` does, + // or the writer rejects the first append. Setting the range directly rather than incrementing + // also covers a genesis that doesn't sit on a static file boundary. + if matches!(receipts_tip, Some(None)) { + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Receipts)?; + writer.user_header_mut().set_block_range(genesis_block_number, genesis_block_number); + writer.commit()?; + } + let to_block = receipts_target.or(to_block); let end = to_block.map_or(Bound::Unbounded, Bound::Included); + let policy = ImportPolicy { headers_tip, is_receipt_verifiable }; while let Some(meta) = rx.recv()? { let meta = meta?; let from = height; let provider = provider_factory.database_provider_rw()?; + // Headers and receipts are separate static file segments, each with its own writer, only + // open the receipts one when `--with-receipts` was requested. + let mut receipts_writer = store_receipts + .then(|| static_file_provider.latest_writer(StaticFileSegment::Receipts)) + .transpose()?; + height = process::( &meta, &mut static_file_provider.latest_writer(StaticFileSegment::Headers)?, + receipts_writer.as_mut(), &provider, hash_collector, (Bound::Included(height), end), + policy, )?; - save_stage_checkpoints(&provider, from, height, height, height)?; + // Drop the receipts writer's lock before `provider.commit()`, which locks every static + // file segment (including receipts) via `has_unwind_queued`. + drop(receipts_writer); + + // A receipts-only backfill trails `headers_tip`; the checkpoints must not regress below + // the headers already in static files. + let checkpoint = height.max(headers_tip); + save_stage_checkpoints(&provider, from, checkpoint, checkpoint, checkpoint)?; provider.commit()?; @@ -241,14 +416,25 @@ where provider.commit()?; + // A repair that stops short leaves the receipts behind the executed range, which the Execution + // stage reports as missing static file data on the next node start. + if let Some(target) = receipts_target && + height < target + { + eyre::bail!( + "receipt repair reached block {height} but the executed range ends at {target}. The \ + source ran out of files, re-run with the files covering blocks {}..={target}", + height + 1, + ); + } + Ok(height) } /// Saves progress of ERA import into stages sync. /// -/// Since the ERA import does the same work as `HeaderStage` and `BodyStage`, it needs to inform -/// these stages that this work has already been done. Otherwise, there might be some conflict with -/// database integrity. +/// Never marks `Execution` done: this import writes no state, so moving that checkpoint would let +/// a node start from blocks whose accounts and storage were never written. pub fn save_stage_checkpoints

( provider: P, from: BlockNumber, @@ -279,22 +465,32 @@ where pub fn process( meta: &(impl EraMeta + ?Sized), writer: &mut StaticFileProviderRWRefMut<'_,

::Primitives>, + receipts_writer: Option< + &mut StaticFileProviderRWRefMut<'_,

::Primitives>, + >, provider: &P, hash_collector: &mut Collector, block_numbers: impl RangeBounds, + policy: ImportPolicy<'_>, ) -> eyre::Result where - S: EraBlockReader, + S: EraBlockReader>, B: Block

, BH: FullBlockHeader + Value, BB: FullBlockBody< Transaction = <

::Primitives as NodePrimitives>::SignedTx, OmmerHeader = BH, >, - P: DBProvider + NodePrimitivesProvider + BlockWriter, + P: DBProvider + + NodePrimitivesProvider + + BlockWriter + + BlockBodyIndicesProvider + + BlockHashReader,

::Primitives: NodePrimitives, + ReceiptOf

: Compact + Receipt, { - let iter = S::blocks(meta)? + let decode_receipts = receipts_writer.is_some(); + let iter = S::blocks(meta, decode_receipts)? .map(Some) .chain(std::iter::once_with(|| match meta.mark_as_processed() { Ok(()) => None, @@ -302,10 +498,28 @@ where })) .flatten(); - process_iter(iter, writer, provider, hash_collector, block_numbers) + process_iter(iter, writer, receipts_writer, provider, hash_collector, block_numbers, policy) +} + +/// Per-block import policy for [`process`] and [`process_iter`]. +#[derive(Clone, Copy)] +pub struct ImportPolicy<'a> { + /// Blocks at or below this are backfilled receipts-only; blocks above it are written in full. + pub headers_tip: BlockNumber, + /// Whether a block's receipts are verified against its header commitments. + pub is_receipt_verifiable: &'a dyn Fn(BlockNumber) -> bool, } -/// Extracts a pair of [`FullBlockHeader`] and [`FullBlockBody`] from [`BlockTuple`]. +impl std::fmt::Debug for ImportPolicy<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + // `is_receipt_verifiable` is a `dyn Fn` and can't be formatted. + f.debug_struct("ImportPolicy") + .field("headers_tip", &self.headers_tip) + .finish_non_exhaustive() + } +} + +/// Extracts a `(header, body)` pair from [`BlockTuple`]. pub fn decode(block: Result) -> eyre::Result<(BH, BB)> where BH: FullBlockHeader + Value, @@ -315,27 +529,103 @@ where let block = block?; let header: BH = block.header.decode()?; let body: BB = block.body.decode()?; - Ok((header, body)) } -/// Extracts block headers and bodies from `iter` and appends them using `writer` and `provider`. +/// Like [`decode`], but also extracts receipts when `decode_receipts` is `true`. `receipts` is +/// `Some` whenever `decode_receipts` is `true`: `era1`'s `CompressedReceipts` entry is mandatory +/// per spec, so (unlike `.ere`) it is never `None` just because the file omits it. +pub fn decode_with_receipts( + block: Result, + decode_receipts: bool, +) -> eyre::Result> +where + BH: FullBlockHeader + Value, + BB: FullBlockBody, + R: RlpDecodableReceipt, + E: From + Error + Send + Sync + 'static, +{ + let block = block?; + let header: BH = block.header.decode()?; + let body: BB = block.body.decode()?; + let number = header.number(); + let receipts = decode_receipts + .then(|| -> eyre::Result<_> { + match block.receipts.decode::>>() { + Ok(receipts) => { + Ok(receipts.into_iter().map(|with_bloom| with_bloom.receipt).collect()) + } + // A status field holding a post-state root doesn't fit the node's receipt type, so + // re-read the entry to tell that apart from a genuinely malformed one. + Err(err) => match block.receipts.decode::>() { + Ok(envelopes) => receipts_from_envelopes(number, envelopes), + Err(_) => Err(err.into()), + }, + } + }) + .transpose()?; + + Ok((header, body, receipts)) +} + +/// Converts decoded ERA receipts into the node's receipt type. +/// +/// Rebuilds them through their canonical encoding, the only construction path every node receipt +/// type supports. Receipts predating Byzantium are rejected: they commit to a post-state root +/// rather than a success status, and nothing else in the file records whether the transaction +/// succeeded, so no node receipt type can represent them. +fn receipts_from_envelopes( + number: BlockNumber, + envelopes: Vec, +) -> eyre::Result> { + for envelope in &envelopes { + if matches!(envelope.status_or_post_state(), Eip658Value::PostState(_)) { + eyre::bail!( + "block {number} has pre-Byzantium receipts, which commit to a post-state root \ + rather than a success status and so cannot be represented by this node's receipt \ + type. Receipt import is only supported from Byzantium onwards" + ); + } + } + + let encoded = alloy_rlp::encode(&envelopes); + + Ok(Vec::>::decode(&mut encoded.as_slice())? + .into_iter() + .map(|with_bloom| with_bloom.receipt) + .collect()) +} + +/// Extracts block headers, bodies and (optionally) receipts from `iter` and appends them using +/// `writer`, `receipts_writer` and `provider`. /// /// Collects hash to height using `hash_collector`. /// /// Skips all blocks below the [`start_bound`] of `block_numbers` and stops when reaching past the /// [`end_bound`] or the end of the file. /// +/// Blocks at or below `headers_tip` only have their receipts backfilled; blocks above it get +/// header, body and receipts written. +/// +/// When `receipts_writer` is `Some`, every block must carry receipts, a block without them, +/// possible for `.ere` files, whose receipts are optional per spec is an error. Receipts are +/// checked against the header's logs bloom, and against its receipts root when +/// `is_receipt_verifiable` returns `true`. +/// /// Returns last block height. /// /// [`start_bound`]: RangeBounds::start_bound /// [`end_bound`]: RangeBounds::end_bound pub fn process_iter( - mut iter: impl Iterator>, + mut iter: impl Iterator>>>, writer: &mut StaticFileProviderRWRefMut<'_,

::Primitives>, + mut receipts_writer: Option< + &mut StaticFileProviderRWRefMut<'_,

::Primitives>, + >, provider: &P, hash_collector: &mut Collector, block_numbers: impl RangeBounds, + policy: ImportPolicy<'_>, ) -> eyre::Result where B: Block

, @@ -344,8 +634,13 @@ where Transaction = <

::Primitives as NodePrimitives>::SignedTx, OmmerHeader = BH, >, - P: DBProvider + NodePrimitivesProvider + BlockWriter, + P: DBProvider + + NodePrimitivesProvider + + BlockWriter + + BlockBodyIndicesProvider + + BlockHashReader,

::Primitives: NodePrimitives, + ReceiptOf

: Compact + Receipt, { let mut last_header_number = match block_numbers.start_bound() { Bound::Included(&number) => number, @@ -359,7 +654,7 @@ where }; for block in &mut iter { - let (header, body) = block?; + let (header, body, receipts) = block?; let number = header.number(); if number <= last_header_number { @@ -382,21 +677,121 @@ where ); } - let hash = header.hash_slow(); last_header_number = number; - // Append to Headers segment - writer.append_header(&header, &hash)?; - - // Write bodies to database. - provider.append_block_bodies(vec![(header.number(), Some(&body))])?; + // Header and body are only written for new blocks, when backfilling receipts onto an + // earlier import the block already has both persisted. + if number > policy.headers_tip { + let hash = header.hash_slow(); + writer.append_header(&header, &hash)?; + provider.append_block_bodies(vec![(header.number(), Some(&body))])?; + hash_collector.insert(hash, number)?; + } else if provider.block_hash(number)? != Some(header.hash_slow()) { + // Receipts are verified against the source header, so it must be the block already + // persisted at this height, not merely a self-consistent one. + eyre::bail!( + "block {number} in this ERA file does not match the block already imported at \ + that height" + ); + } - hash_collector.insert(hash, number)?; + if let Some(receipts_writer) = receipts_writer.as_deref_mut() { + if let Some(receipts) = receipts.as_deref() { + verify_receipts(&header, receipts, (policy.is_receipt_verifiable)(number))?; + } + provider.write_block_receipts(receipts_writer, number, receipts)?; + } } Ok(last_header_number) } +/// Checks a block's receipts against its header. +/// +/// The bloom is always checked, the root only when `check_root` is set, as pre-Byzantium receipts +/// can't be recomputed here. +fn verify_receipts(header: &BH, receipts: &[R], check_root: bool) -> eyre::Result<()> +where + BH: FullBlockHeader, + R: Receipt, +{ + let with_bloom = receipts.iter().map(TxReceipt::with_bloom_ref).collect::>(); + let logs_bloom = with_bloom.iter().fold(Bloom::ZERO, |bloom, r| bloom | r.bloom_ref()); + + if logs_bloom != header.logs_bloom() { + eyre::bail!("logs bloom mismatch for block {}", header.number()); + } + + if check_root { + let receipts_root = calculate_receipt_root(&with_bloom); + if receipts_root != header.receipts_root() { + eyre::bail!( + "receipts root mismatch for block {}: computed {receipts_root}, header has {}", + header.number(), + header.receipts_root(), + ); + } + } + + Ok(()) +} + +/// Lets a provider append one block's receipts to a `Receipts` static file writer, using itself +/// to look up the block's transaction range. +trait BlockReceiptsWriterExt: BlockBodyIndicesProvider { + /// Appends `receipts` for block `number` to `receipts_writer`. + /// + /// Errors if the block has no receipts, or if the receipt count doesn't match the block's + /// transaction count. + fn write_block_receipts( + &self, + receipts_writer: &mut StaticFileProviderRWRefMut<'_, N>, + number: BlockNumber, + receipts: Option>, + ) -> eyre::Result<()> + where + N::Receipt: Compact; +} + +impl BlockReceiptsWriterExt for P { + fn write_block_receipts( + &self, + receipts_writer: &mut StaticFileProviderRWRefMut<'_, N>, + number: BlockNumber, + receipts: Option>, + ) -> eyre::Result<()> + where + N::Receipt: Compact, + { + let Some(block_receipts) = receipts else { + eyre::bail!( + "block {number} has no receipts in the imported ERA file; drop --with-receipts, \ + or import files that carry receipts for every block (`.era1` always includes \ + them; `.ere` receipts are optional per spec)" + ); + }; + + let indices = self + .block_body_indices(number)? + .ok_or_else(|| eyre::eyre!("missing block body indices for block {number}"))?; + + if block_receipts.len() as u64 != indices.tx_count { + eyre::bail!( + "receipt count mismatch for block {number}: {} receipt(s) for {} transaction(s)", + block_receipts.len(), + indices.tx_count, + ); + } + + receipts_writer.increment_block(number)?; + receipts_writer.append_receipts( + (indices.first_tx_num..).zip(block_receipts.iter()).map(Ok::<_, ProviderError>), + )?; + + Ok(()) + } +} + /// Dumps the contents of `hash_collector` into [`tables::HeaderNumbers`]. pub fn build_index

( provider: &P, @@ -475,25 +870,152 @@ where #[cfg(test)] mod tests { use super::*; - use alloy_consensus::Header; + use alloy_consensus::{Header, Receipt as RlpReceipt, ReceiptWithBloom, TxLegacy, TxType}; + use alloy_primitives::{Address, Bytes, Log, Signature, B256}; use reth_db_common::init::init_genesis; - use reth_ethereum_primitives::{Block, BlockBody}; + use reth_era::{ + era1::types::execution::{ + CompressedBody, CompressedHeader, CompressedReceipts, TotalDifficulty, + }, + ere::types::execution::{ + CompressedBody as EreCompressedBody, CompressedHeader as EreCompressedHeader, + CompressedSlimReceipts, SlimReceipt, + }, + }; + use reth_ethereum_primitives::{Block, BlockBody, Receipt, TransactionSigned}; use reth_provider::{ - test_utils::create_test_provider_factory, DatabaseProviderFactory, - StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, + test_utils::{ + create_test_provider_factory, create_test_provider_factory_with_genesis_block_number, + }, + DatabaseProviderFactory, ReceiptProvider, StageCheckpointReader, StaticFileProviderFactory, + StaticFileSegment, StaticFileWriter, }; + use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig}; use std::{cell::Cell, path::Path}; use tempfile::tempdir; + /// Builds an `.era1` block tuple for a transaction-less block carrying `receipts`. + fn era1_block_tuple(number: u64, receipts: Vec) -> BlockTuple { + let header = Header { number, ..Default::default() }; + BlockTuple::new( + CompressedHeader::from_rlp(&alloy_rlp::encode(&header)).unwrap(), + CompressedBody::from_rlp(&alloy_rlp::encode(BlockBody::default())).unwrap(), + CompressedReceipts::from_rlp(&alloy_rlp::encode(&receipts)).unwrap(), + TotalDifficulty::new(U256::ZERO), + ) + } + + /// Builds a block with one transaction, its receipt, and a header committing to that receipt. + fn block_with_one_receipt(number: u64) -> (Header, BlockBody, Receipt) { + let tx = TransactionSigned::new_unhashed( + TxLegacy::default().into(), + Signature::test_signature(), + ); + let receipt = Receipt { + tx_type: TxType::Legacy, + success: true, + cumulative_gas_used: 21_000, + logs: vec![], + }; + + let with_bloom = vec![TxReceipt::with_bloom_ref(&receipt)]; + let header = Header { + number, + receipts_root: calculate_receipt_root(&with_bloom), + logs_bloom: with_bloom.iter().fold(Bloom::ZERO, |bloom, r| bloom | r.bloom_ref()), + ..Default::default() + }; + + (header, BlockBody { transactions: vec![tx], ..Default::default() }, receipt) + } + + /// Wraps `status` in a legacy `.era1` receipt envelope. + fn era1_receipt(status: Eip658Value) -> ReceiptEnvelope { + ReceiptEnvelope::Legacy(ReceiptWithBloom::new( + RlpReceipt { status, cumulative_gas_used: 21_000, logs: vec![] }, + Bloom::ZERO, + )) + } + + /// Builds an `.ere` block tuple carrying canonical slim receipts. + fn ere_block_tuple(number: u64, receipts: &[SlimReceipt]) -> EreBlockTuple { + let header = Header { number, ..Default::default() }; + EreBlockTuple::new( + EreCompressedHeader::from_rlp(&alloy_rlp::encode(&header)).unwrap(), + EreCompressedBody::from_rlp(&alloy_rlp::encode(BlockBody::default())).unwrap(), + ) + .with_receipts(CompressedSlimReceipts::from_receipts(receipts).unwrap()) + } + struct TestEra; - impl EraBlockReader for TestEra { + impl EraBlockReader for TestEra { fn blocks( _meta: &M, - ) -> eyre::Result>> { - Ok([1, 2] - .into_iter() - .map(|number| Ok((Header { number, ..Default::default() }, BlockBody::default())))) + _decode_receipts: bool, + ) -> eyre::Result>)>>> + { + Ok([1, 2].into_iter().map(|number| { + Ok((Header { number, ..Default::default() }, BlockBody::default(), None)) + })) + } + } + + /// Like [`TestEra`], but yields `Some(vec![])` receipts for each (transaction-less) block. + struct TestEraWithEmptyReceipts; + + impl EraBlockReader for TestEraWithEmptyReceipts { + fn blocks( + _meta: &M, + _decode_receipts: bool, + ) -> eyre::Result>)>>> + { + Ok([1, 2].into_iter().map(|number| { + Ok((Header { number, ..Default::default() }, BlockBody::default(), Some(vec![]))) + })) + } + } + + /// Empty receipts for the two blocks immediately after a genesis at block 100. + struct TestEraWithNonZeroGenesis; + + impl EraBlockReader for TestEraWithNonZeroGenesis { + fn blocks( + _meta: &M, + _decode_receipts: bool, + ) -> eyre::Result>)>>> + { + Ok([101, 102].into_iter().map(|number| { + Ok((Header { number, ..Default::default() }, BlockBody::default(), Some(vec![]))) + })) + } + } + + /// Like [`TestEra`], but yields a single receipt for a transaction-less block, provoking a + /// receipt/transaction count mismatch. + struct TestEraWithMismatchedReceipts; + + impl EraBlockReader + for TestEraWithMismatchedReceipts + { + fn blocks( + _meta: &M, + _decode_receipts: bool, + ) -> eyre::Result< + impl Iterator< + Item = eyre::Result>, + >, + > { + Ok(std::iter::once(Ok(( + Header { number: 1, ..Default::default() }, + BlockBody::default(), + Some(vec![reth_ethereum_primitives::Receipt { + tx_type: alloy_consensus::TxType::Legacy, + success: true, + cumulative_gas_used: 0, + logs: vec![], + }]), + )))) } } @@ -527,13 +1049,327 @@ mod tests { Ok(TestMeta { marked: Cell::new(false) }), ]); - let height = - import::(stream, &pf, &mut hash_collector, Some(1)) - .unwrap(); + let height = import::( + stream, + &pf, + &mut hash_collector, + Some(1), + false, + &|_| false, + ) + .unwrap(); assert_eq!(height, 1); } + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn backfill_does_not_move_header_checkpoints_backwards() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let static_file_provider = pf.static_file_provider(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + // Import headers and bodies for blocks 1 and 2 first. + let provider = pf.database_provider_rw().unwrap(); + { + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let meta = TestMeta { marked: Cell::new(false) }; + process::( + &meta, + &mut writer, + None, + &provider, + &mut hash_collector, + 0..=2, + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| false }, + ) + .unwrap(); + writer.commit().unwrap(); + } + save_stage_checkpoints(&provider, 0, 2, 2, 2).unwrap(); + // Only block 1 was executed, so the repair ends below the headers already in static files. + provider.save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(1)).unwrap(); + provider.commit().unwrap(); + + let stream = futures_util::stream::iter(vec![Ok(TestMeta { marked: Cell::new(false) })]); + let height = import::( + stream, + &pf, + &mut hash_collector, + None, + true, + &|_| true, + ) + .unwrap(); + + assert_eq!(height, 1); + let provider = pf.database_provider_rw().unwrap(); + for stage in [StageId::Headers, StageId::Bodies] { + assert_eq!( + provider.get_stage_checkpoint(stage).unwrap().map(|c| c.block_number), + Some(2), + "{stage} checkpoint must not regress below the headers already in static files" + ); + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn receipt_import_requires_an_executed_range() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + // Nothing executed, so there is no range whose receipts could survive a node start. + let stream = futures_util::stream::iter(vec![Ok(TestMeta { marked: Cell::new(false) })]); + let result = import::( + stream, + &pf, + &mut hash_collector, + None, + true, + &|_| true, + ); + + assert!(result.is_err()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn receipt_import_rejects_a_to_block_off_the_execution_checkpoint() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + let provider = pf.database_provider_rw().unwrap(); + provider.save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(2)).unwrap(); + provider.commit().unwrap(); + + // Stopping at 1 would leave the receipts behind the executed range. + let stream = futures_util::stream::iter(vec![Ok(TestMeta { marked: Cell::new(false) })]); + let result = import::( + stream, + &pf, + &mut hash_collector, + Some(1), + true, + &|_| true, + ); + + assert!(result.is_err()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn receipt_import_backfills_a_completely_absent_segment() { + // No genesis, so the Receipts segment does not exist at all. + let pf = create_test_provider_factory(); + let static_file_provider = pf.static_file_provider(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + // Stand in for the genesis entries `init_genesis` would write to every segment, leaving + // only `Receipts` absent. + { + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Transactions).unwrap(); + writer.increment_block(0).unwrap(); + writer.commit().unwrap(); + } + + let provider = pf.database_provider_rw().unwrap(); + { + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let genesis = Header::default(); + writer.append_header(&genesis, &genesis.hash_slow()).unwrap(); + + let meta = TestMeta { marked: Cell::new(false) }; + process::( + &meta, + &mut writer, + None, + &provider, + &mut hash_collector, + 0..=2, + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| true }, + ) + .unwrap(); + writer.commit().unwrap(); + } + provider.save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(2)).unwrap(); + provider.commit().unwrap(); + + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts), + None + ); + + let stream = futures_util::stream::iter(vec![Ok(TestMeta { marked: Cell::new(false) })]); + let height = import::( + stream, + &pf, + &mut hash_collector, + None, + true, + &|_| true, + ) + .unwrap(); + + assert_eq!(height, 2); + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts), + Some(2) + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn receipt_import_backfills_an_absent_segment_after_non_zero_genesis() { + const GENESIS: u64 = 100; + + let pf = create_test_provider_factory_with_genesis_block_number(GENESIS); + let static_file_provider = pf.static_file_provider(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + { + let mut writer = + static_file_provider.get_writer(GENESIS, StaticFileSegment::Transactions).unwrap(); + writer.user_header_mut().set_block_range(GENESIS, GENESIS); + writer.commit().unwrap(); + } + + let provider = pf.database_provider_rw().unwrap(); + { + let mut writer = + static_file_provider.get_writer(GENESIS, StaticFileSegment::Headers).unwrap(); + let genesis = Header { number: GENESIS, ..Default::default() }; + writer.user_header_mut().set_block_range(GENESIS, GENESIS); + writer + .append_header_direct(&genesis, genesis.difficulty, &genesis.hash_slow()) + .unwrap(); + + let meta = TestMeta { marked: Cell::new(false) }; + process::( + &meta, + &mut writer, + None, + &provider, + &mut hash_collector, + GENESIS..=102, + ImportPolicy { headers_tip: GENESIS, is_receipt_verifiable: &|_| true }, + ) + .unwrap(); + writer.commit().unwrap(); + } + provider.save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(102)).unwrap(); + provider.commit().unwrap(); + + assert_eq!(static_file_provider.genesis_block_number(), GENESIS); + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts), + None + ); + + let stream = futures_util::stream::iter(vec![Ok(TestMeta { marked: Cell::new(false) })]); + let height = import::( + stream, + &pf, + &mut hash_collector, + None, + true, + &|_| true, + ) + .unwrap(); + + assert_eq!(height, 102); + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts), + Some(102) + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn receipt_import_rejects_a_prune_config_that_bypasses_static_files() { + let prune_modes = PruneModes { + receipts_log_filter: ReceiptsLogPruneConfig( + std::iter::once((Address::ZERO, PruneMode::Full)).collect(), + ), + ..Default::default() + }; + let pf = create_test_provider_factory().with_prune_modes(prune_modes); + init_genesis(&pf).unwrap(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + let provider = pf.database_provider_rw().unwrap(); + provider.save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(2)).unwrap(); + provider.commit().unwrap(); + + let stream = futures_util::stream::iter(vec![Ok(TestMeta { marked: Cell::new(false) })]); + let result = import::( + stream, + &pf, + &mut hash_collector, + None, + true, + &|_| true, + ); + + assert!(result.is_err()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn receipt_import_rejects_a_repair_starting_before_byzantium() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + let provider = pf.database_provider_rw().unwrap(); + provider.save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(2)).unwrap(); + provider.commit().unwrap(); + + // Receipts resume at block 1, before this chain's Byzantium activation at block 2. + let stream = futures_util::stream::iter(vec![Ok(TestMeta { marked: Cell::new(false) })]); + let result = import::( + stream, + &pf, + &mut hash_collector, + None, + true, + &|number| number >= 2, + ); + + assert!(result.is_err()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn receipt_import_errors_when_the_source_stops_short() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + let provider = pf.database_provider_rw().unwrap(); + provider.save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(10)).unwrap(); + provider.commit().unwrap(); + + // The source only carries blocks 1 and 2, far short of the executed range. + let stream = futures_util::stream::iter(vec![Ok(TestMeta { marked: Cell::new(false) })]); + let result = import::( + stream, + &pf, + &mut hash_collector, + None, + true, + &|_| true, + ); + + assert!(result.is_err()); + } + #[test] fn process_does_not_mark_partially_consumed_file_processed() { let pf = create_test_provider_factory(); @@ -549,9 +1385,11 @@ mod tests { let height = process::( &meta, &mut writer, + None, &provider, &mut hash_collector, 0..=1, + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| false }, ) .unwrap(); @@ -572,18 +1410,458 @@ mod tests { // Genesis DB sits at height 0, but the first block is 5: a gap that must be rejected // rather than appended (as a pre-merge `.era` import would otherwise produce). - let blocks = [5u64, 6] - .into_iter() - .map(|number| Ok((Header { number, ..Default::default() }, BlockBody::default()))); + let blocks = [5u64, 6].into_iter().map(|number| { + Ok((Header { number, ..Default::default() }, BlockBody::default(), None)) + }); let result = process_iter::<_, Block, _, _>( blocks, &mut writer, + None, + &provider, + &mut hash_collector, + 0.., + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| false }, + ); + + assert!(result.is_err()); + } + + #[test] + fn process_writes_receipts_when_requested() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + + let static_file_provider = pf.static_file_provider(); + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let mut receipts_writer = + static_file_provider.latest_writer(StaticFileSegment::Receipts).unwrap(); + let provider = pf.database_provider_rw().unwrap(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + let meta = TestMeta { marked: Cell::new(false) }; + + let height = process::( + &meta, + &mut writer, + Some(&mut receipts_writer), + &provider, + &mut hash_collector, + 0..=1, + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| false }, + ) + .unwrap(); + receipts_writer.commit().unwrap(); + + assert_eq!(height, 1); + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts), + Some(1) + ); + } + + #[test] + fn process_iter_errors_when_receipts_missing() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + + let static_file_provider = pf.static_file_provider(); + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let mut receipts_writer = + static_file_provider.latest_writer(StaticFileSegment::Receipts).unwrap(); + let provider = pf.database_provider_rw().unwrap(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + // `TestEra` never yields receipts; requesting them must fail rather than silently import + // headers/bodies without them. + let blocks: Vec< + eyre::Result>, + > = vec![Ok((Header { number: 1, ..Default::default() }, BlockBody::default(), None))]; + + let result = process_iter::<_, Block, _, _>( + blocks.into_iter(), + &mut writer, + Some(&mut receipts_writer), + &provider, + &mut hash_collector, + 0.., + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| false }, + ); + + assert!(result.is_err()); + } + + #[test] + fn process_errors_on_receipt_count_mismatch() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + + let static_file_provider = pf.static_file_provider(); + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let mut receipts_writer = + static_file_provider.latest_writer(StaticFileSegment::Receipts).unwrap(); + let provider = pf.database_provider_rw().unwrap(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + let meta = TestMeta { marked: Cell::new(false) }; + + // One receipt for a transaction-less body: a count mismatch that must be rejected rather + // than silently misaligning the Receipts static file against Transactions. + let result = process::( + &meta, + &mut writer, + Some(&mut receipts_writer), + &provider, + &mut hash_collector, + 0.., + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| false }, + ); + + assert!(result.is_err()); + } + + #[test] + fn save_stage_checkpoints_leaves_execution_unset() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let provider = pf.database_provider_rw().unwrap(); + + let execution_before = provider.get_stage_checkpoint(StageId::Execution).unwrap(); + save_stage_checkpoints(&provider, 0, 10, 10, 10).unwrap(); + + assert_eq!( + provider.get_stage_checkpoint(StageId::Headers).unwrap().map(|c| c.block_number), + Some(10) + ); + assert_eq!( + provider.get_stage_checkpoint(StageId::Bodies).unwrap().map(|c| c.block_number), + Some(10) + ); + // Receipt import doesn't produce state, so it must not advance Execution. + assert_eq!(provider.get_stage_checkpoint(StageId::Execution).unwrap(), execution_before); + } + + #[test] + fn backfills_receipts_onto_existing_headers() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let static_file_provider = pf.static_file_provider(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + // First pass: import headers and bodies only, no receipts. + { + let provider = pf.database_provider_rw().unwrap(); + // The writer holds the Headers segment lock, which `provider.commit()` also takes when + // it finalizes every segment, so it must be released first. + { + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let meta = TestMeta { marked: Cell::new(false) }; + process::( + &meta, + &mut writer, + None, + &provider, + &mut hash_collector, + 0..=2, + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| false }, + ) + .unwrap(); + writer.commit().unwrap(); + } + provider.commit().unwrap(); + } + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers), + Some(2) + ); + // Receipts still only cover genesis; blocks 1 and 2 have headers but no receipts yet. + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts), + Some(0) + ); + + // Second pass: backfill receipts for the already-imported headers (`headers_tip = 2`). + { + let provider = pf.database_provider_rw().unwrap(); + { + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let mut receipts_writer = + static_file_provider.latest_writer(StaticFileSegment::Receipts).unwrap(); + let meta = TestMeta { marked: Cell::new(false) }; + process::( + &meta, + &mut writer, + Some(&mut receipts_writer), + &provider, + &mut hash_collector, + 0.., + ImportPolicy { headers_tip: 2, is_receipt_verifiable: &|_| false }, + ) + .unwrap(); + receipts_writer.commit().unwrap(); + writer.commit().unwrap(); + } + provider.commit().unwrap(); + } + + // Receipts now cover the previously header-only range, and headers weren't re-appended. + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts), + Some(2) + ); + assert_eq!( + static_file_provider.get_highest_static_file_block(StaticFileSegment::Headers), + Some(2) + ); + } + + #[test] + fn process_iter_persists_verified_receipts() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let static_file_provider = pf.static_file_provider(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + let (header, body, receipt) = block_with_one_receipt(1); + let provider = pf.database_provider_rw().unwrap(); + { + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let mut receipts_writer = + static_file_provider.latest_writer(StaticFileSegment::Receipts).unwrap(); + + process_iter::<_, Block, _, _>( + std::iter::once(Ok((header, body, Some(vec![receipt.clone()])))), + &mut writer, + Some(&mut receipts_writer), + &provider, + &mut hash_collector, + 0.., + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| true }, + ) + .unwrap(); + + receipts_writer.commit().unwrap(); + writer.commit().unwrap(); + } + provider.commit().unwrap(); + + let provider = pf.provider().unwrap(); + assert_eq!(provider.receipts_by_block(1.into()).unwrap(), Some(vec![receipt])); + } + + #[test] + fn process_iter_rejects_receipts_the_header_does_not_commit_to() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let static_file_provider = pf.static_file_provider(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + // Right receipt count, wrong contents: only the recomputed root catches this. + let (header, body, receipt) = block_with_one_receipt(1); + let tampered = Receipt { cumulative_gas_used: 42_000, ..receipt }; + + let provider = pf.database_provider_rw().unwrap(); + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let mut receipts_writer = + static_file_provider.latest_writer(StaticFileSegment::Receipts).unwrap(); + + let result = process_iter::<_, Block, _, _>( + std::iter::once(Ok((header, body, Some(vec![tampered])))), + &mut writer, + Some(&mut receipts_writer), &provider, &mut hash_collector, 0.., + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| true }, ); assert!(result.is_err()); } + + #[test] + fn decodes_post_byzantium_era1_receipts() { + let tuple = era1_block_tuple(4_370_000, vec![era1_receipt(Eip658Value::Eip658(true))]); + + let (_, _, receipts) = + decode_with_receipts::(Ok(tuple), true).unwrap(); + + let receipts = receipts.unwrap(); + assert_eq!(receipts.len(), 1); + assert!(receipts[0].success); + assert_eq!(receipts[0].cumulative_gas_used, 21_000); + } + + #[test] + fn decodes_typed_ere_receipts_into_node_receipts() { + let logs = vec![Log::new_unchecked( + Address::repeat_byte(0x11), + vec![B256::repeat_byte(0x22)], + Bytes::from_static(b"typed"), + )]; + let slim = vec![ + SlimReceipt { + tx_type: TxType::Eip2930, + status: Eip658Value::Eip658(true), + cumulative_gas_used: 21_000, + logs: logs.clone(), + }, + SlimReceipt { + tx_type: TxType::Eip1559, + status: Eip658Value::Eip658(false), + cumulative_gas_used: 42_000, + logs: vec![], + }, + ]; + let tuple = ere_block_tuple(12_965_000, &slim); + + let (_, _, receipts) = + Ere::decode_with_receipts::(Ok(tuple), true) + .unwrap(); + + assert_eq!( + receipts, + Some(vec![ + Receipt { + tx_type: TxType::Eip2930, + success: true, + cumulative_gas_used: 21_000, + logs, + }, + Receipt { + tx_type: TxType::Eip1559, + success: false, + cumulative_gas_used: 42_000, + logs: vec![], + }, + ]) + ); + } + + #[test] + fn rejects_pre_byzantium_era1_receipts() { + // Mainnet's first transaction: its receipt commits to a post-state root. + let tuple = era1_block_tuple( + 46_147, + vec![era1_receipt(Eip658Value::PostState(B256::repeat_byte(1)))], + ); + + let err = decode_with_receipts::(Ok(tuple), true) + .unwrap_err() + .to_string(); + + assert!(err.contains("pre-Byzantium"), "unexpected error: {err}"); + assert!(err.contains("46147"), "error should name the offending block: {err}"); + } + + #[test] + fn skips_pre_byzantium_receipts_when_not_requested() { + // A header-only import of the same file must stay unaffected. + let tuple = era1_block_tuple( + 46_147, + vec![era1_receipt(Eip658Value::PostState(B256::repeat_byte(1)))], + ); + + let (header, _, receipts) = + decode_with_receipts::(Ok(tuple), false).unwrap(); + + assert_eq!(header.number, 46_147); + assert!(receipts.is_none()); + } + + #[test] + fn backfill_rejects_a_header_that_differs_from_the_persisted_one() { + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + let static_file_provider = pf.static_file_provider(); + let folder = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(folder.path().to_owned())); + + let provider = pf.database_provider_rw().unwrap(); + { + let mut writer = + static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let meta = TestMeta { marked: Cell::new(false) }; + process::( + &meta, + &mut writer, + None, + &provider, + &mut hash_collector, + 0..=2, + ImportPolicy { headers_tip: 0, is_receipt_verifiable: &|_| false }, + ) + .unwrap(); + writer.commit().unwrap(); + } + provider.commit().unwrap(); + + // Same height and receipt count as the persisted block, but a different header. + let provider = pf.database_provider_rw().unwrap(); + let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers).unwrap(); + let mut receipts_writer = + static_file_provider.latest_writer(StaticFileSegment::Receipts).unwrap(); + + let result = process_iter::<_, Block, _, _>( + std::iter::once(Ok(( + Header { number: 1, gas_limit: 42, ..Default::default() }, + BlockBody::default(), + Some(vec![]), + ))), + &mut writer, + Some(&mut receipts_writer), + &provider, + &mut hash_collector, + 0.., + ImportPolicy { headers_tip: 2, is_receipt_verifiable: &|_| false }, + ); + + assert!(result.is_err()); + } + + #[test] + fn verify_receipts_rejects_tampered_contents() { + let receipts = vec![Receipt { + tx_type: TxType::Legacy, + success: true, + cumulative_gas_used: 21_000, + logs: vec![], + }]; + + // Commit the header to the receipts as decoded. + let with_bloom = receipts.iter().map(TxReceipt::with_bloom_ref).collect::>(); + let header = Header { + receipts_root: calculate_receipt_root(&with_bloom), + logs_bloom: with_bloom.iter().fold(Bloom::ZERO, |bloom, r| bloom | r.bloom_ref()), + ..Default::default() + }; + verify_receipts(&header, &receipts, true).unwrap(); + + // Same receipt count, different contents: the recomputed root no longer matches. + let tampered = vec![Receipt { cumulative_gas_used: 42_000, ..receipts[0].clone() }]; + assert!(verify_receipts(&header, &tampered, true).is_err()); + } + + #[test] + fn verify_receipts_checks_the_bloom_even_without_the_root() { + let receipts = vec![Receipt { + tx_type: TxType::Legacy, + success: true, + cumulative_gas_used: 21_000, + logs: vec![Log::new_unchecked( + Address::ZERO, + vec![B256::repeat_byte(1)], + Bytes::default(), + )], + }]; + + // The header commits to no logs at all, so the bloom cannot match. + assert!(verify_receipts(&Header::default(), &receipts, false).is_err()); + } } diff --git a/crates/era-utils/src/lib.rs b/crates/era-utils/src/lib.rs index c6df4e1f980..e83ac34c169 100644 --- a/crates/era-utils/src/lib.rs +++ b/crates/era-utils/src/lib.rs @@ -9,6 +9,6 @@ mod export; pub use export::{export, EraBlockWriter, ExportBlock, ExportConfig}; pub use history::{ - build_index, calculate_td_by_number, decode, import, open, process, process_iter, - save_stage_checkpoints, Era, Era1, EraBlockReader, Ere, + build_index, calculate_td_by_number, decode, decode_with_receipts, import, open, process, + process_iter, save_stage_checkpoints, Era, Era1, EraBlockReader, Ere, ImportPolicy, }; diff --git a/crates/era-utils/tests/it/history.rs b/crates/era-utils/tests/it/history.rs index 466381abeaf..2a7955caa49 100644 --- a/crates/era-utils/tests/it/history.rs +++ b/crates/era-utils/tests/it/history.rs @@ -6,7 +6,13 @@ use reth_era_downloader::{EraClient, EraStream, EraStreamConfig}; use reth_era_utils::{export, import, Era1, Ere, ExportConfig}; use reth_etl::Collector; use reth_fs_util as fs; -use reth_provider::{test_utils::create_test_provider_factory, BlockNumReader, BlockReader}; +use reth_primitives_traits::Block as _; +use reth_provider::{ + test_utils::create_test_provider_factory, BlockNumReader, BlockReader, DBProvider, + DatabaseProviderFactory, ReceiptProvider, StageCheckpointWriter, StaticFileProviderFactory, +}; +use reth_stages_types::{StageCheckpoint, StageId}; +use reth_static_file_types::StaticFileSegment; use std::str::FromStr; use tempfile::tempdir; @@ -39,11 +45,81 @@ async fn test_history_imports_from_fresh_state_successfully() { let expected_block_number = 8191; let actual_block_number = - import::(stream, &pf, &mut hash_collector, None).unwrap(); + import::(stream, &pf, &mut hash_collector, None, false, &|_| false) + .unwrap(); assert_eq!(actual_block_number, expected_block_number); } +/// Repairing the `Receipts` segment of an executed range from a real `.era1` file. +/// +/// Mainnet's first transaction is in block 46147, so every block in the first era1 file is +/// transaction-less: this exercises the writer plumbing, not receipt decoding. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn test_history_import_with_receipts() { + let url = Url::from_str(ITHACA_ERA_INDEX_URL).unwrap(); + let expected_height = 8191; + + let pf = create_test_provider_factory(); + init_genesis(&pf).unwrap(); + + let collector_dir = tempdir().unwrap(); + let mut hash_collector = Collector::new(4096, Some(collector_dir.path().to_owned())); + + // Headers and bodies first, standing in for a node that has synced and executed this range. + let folder = tempdir().unwrap(); + let client = EraClient::new(ClientWithFakeIndex(Client::new()), url.clone(), folder.path()); + let config = EraStreamConfig::default().with_max_files(1).with_max_concurrent_downloads(1); + let imported_height = import::( + EraStream::new(client, config), + &pf, + &mut hash_collector, + None, + false, + &|_| true, + ) + .unwrap(); + assert_eq!(imported_height, expected_height); + + let provider = pf.database_provider_rw().unwrap(); + provider + .save_stage_checkpoint(StageId::Execution, StageCheckpoint::new(expected_height)) + .unwrap(); + provider.commit().unwrap(); + + // Receipts are then repaired over the executed range. + let folder = tempdir().unwrap(); + let client = EraClient::new(ClientWithFakeIndex(Client::new()), url, folder.path()); + let config = EraStreamConfig::default().with_max_files(1).with_max_concurrent_downloads(1); + let repaired_height = import::( + EraStream::new(client, config), + &pf, + &mut hash_collector, + None, + true, + &|_| true, + ) + .unwrap(); + assert_eq!(repaired_height, expected_height); + + assert_eq!( + pf.static_file_provider().get_highest_static_file_block(StaticFileSegment::Receipts), + Some(expected_height), + "receipts static file should cover the executed range" + ); + + let provider = pf.provider().unwrap(); + for &block_num in &[0, 1, 100, 8191] { + let block = provider.block_by_number(block_num).unwrap().unwrap(); + let receipts = provider.receipts_by_block(block_num.into()).unwrap().unwrap(); + assert_eq!( + receipts.len(), + block.body().transactions().count(), + "block {block_num} should have one receipt per transaction" + ); + } +} + /// Test that verifies the complete roundtrip from importing to exporting era1 files. /// It validates : /// - Downloads the first era1 file from ithaca's url and import the file data, into the database @@ -69,7 +145,8 @@ async fn test_roundtrip_export_after_import() { // Import blocks from one era1 file into database let last_imported_block_height = - import::(stream, &pf, &mut hash_collector, None).unwrap(); + import::(stream, &pf, &mut hash_collector, None, false, &|_| false) + .unwrap(); assert_eq!(last_imported_block_height, 8191); let provider_ref = pf.provider_rw().unwrap().0; @@ -179,7 +256,8 @@ async fn test_ere_roundtrip_export_after_import() { let mut hash_collector = Collector::new(4096, Some(collector_dir.path().to_owned())); let imported_height = - import::(stream, &pf, &mut hash_collector, None).unwrap(); + import::(stream, &pf, &mut hash_collector, None, false, &|_| false) + .unwrap(); assert_eq!(imported_height, 8191); let provider_ref = pf.provider_rw().unwrap().0; @@ -233,9 +311,15 @@ async fn test_ere_roundtrip_export_after_import() { // The exported files are returned in ascending block order, which is the order the importer // expects. let stream = futures_util::stream::iter(ere_files.into_iter().map(|p| Ok(FileMeta::new(p)))); - let reimported_height = - import::(stream, &reimport_pf, &mut reimport_collector, None) - .unwrap(); + let reimported_height = import::( + stream, + &reimport_pf, + &mut reimport_collector, + None, + false, + &|_| true, + ) + .unwrap(); assert_eq!( reimported_height, EXPORT_LAST_BLOCK, diff --git a/crates/era/src/ere/types/execution.rs b/crates/era/src/ere/types/execution.rs index 86fd1b4f056..4f8e853d4cc 100644 --- a/crates/era/src/ere/types/execution.rs +++ b/crates/era/src/ere/types/execution.rs @@ -17,7 +17,7 @@ use crate::{ }, e2s::{error::E2sError, types::Entry}, }; -use alloy_consensus::{Block, BlockBody, Eip658Value, Header, TxType}; +use alloy_consensus::{Block, BlockBody, Eip658Value, Header, Receipt, ReceiptEnvelope, TxType}; use alloy_primitives::{Log, B256, U256}; use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable}; use sha2::{Digest, Sha256}; @@ -260,6 +260,16 @@ pub struct SlimReceipt { pub logs: Vec, } +impl From for ReceiptEnvelope { + /// Restores the bloom that the slim form omits, recomputing it from the logs. + fn from(receipt: SlimReceipt) -> Self { + let SlimReceipt { tx_type, status, cumulative_gas_used, logs } = receipt; + let receipt = Receipt { status, cumulative_gas_used, logs }; + + Self::from_typed(tx_type, receipt.with_bloom()) + } +} + /// Proof type discriminant used inside the Proof entry's RLP envelope. /// /// Maps to specific Portal Network proof objects. diff --git a/crates/stages/stages/src/stages/era.rs b/crates/stages/stages/src/stages/era.rs index f4ce4bed17f..16cff9fa65b 100644 --- a/crates/stages/stages/src/stages/era.rs +++ b/crates/stages/stages/src/stages/era.rs @@ -231,12 +231,18 @@ where // ascending order let mut writer = static_file_provider.latest_writer(StaticFileSegment::Headers)?; + // This stage never persists receipts, execution does that separately. let height = era::process_iter( - era, + era.map(|r| r.map(|(header, body)| (header, body, None))), &mut writer, + None, provider, &mut self.hash_collector, last_header_number..=input.target(), + era::ImportPolicy { + headers_tip: last_header_number, + is_receipt_verifiable: &|_| false, + }, ) .map_err(|e| StageError::Fatal(e.into()))?; diff --git a/crates/storage/provider/src/test_utils/mod.rs b/crates/storage/provider/src/test_utils/mod.rs index 01c5f63c995..8a5df8d1ea7 100644 --- a/crates/storage/provider/src/test_utils/mod.rs +++ b/crates/storage/provider/src/test_utils/mod.rs @@ -1,9 +1,12 @@ use crate::{ - providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBBuilder, StaticFileProvider}, + providers::{ + NodeTypesForProvider, ProviderNodeTypes, RocksDBBuilder, StaticFileProvider, + StaticFileProviderBuilder, + }, HashingWriter, ProviderFactory, TrieWriter, }; use alloy_primitives::B256; -use reth_chainspec::{ChainSpec, MAINNET}; +use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET}; use reth_db::{mdbx::DatabaseArguments, test_utils::TempDatabase, DatabaseEnv}; use reth_errors::ProviderResult; use reth_ethereum_engine_primitives::EthEngineTypes; @@ -49,12 +52,33 @@ pub fn create_test_provider_factory() -> ProviderFactory { pub fn create_test_provider_factory_with_chain_spec( chain_spec: Arc, ) -> ProviderFactory { - create_test_provider_factory_with_node_types::(chain_spec) + let genesis_block_number = chain_spec.genesis.number.unwrap_or_default(); + create_test_provider_factory_with_node_types_and_genesis::( + chain_spec, + genesis_block_number, + ) +} + +/// Creates a test provider factory whose chain starts at `genesis_block_number`. +pub fn create_test_provider_factory_with_genesis_block_number( + genesis_block_number: u64, +) -> ProviderFactory { + let mut genesis = MAINNET.genesis.clone(); + genesis.number = Some(genesis_block_number); + let chain_spec = Arc::new(ChainSpecBuilder::mainnet().genesis(genesis).build()); + create_test_provider_factory_with_chain_spec(chain_spec) } /// Creates test provider factory with provided chain spec. pub fn create_test_provider_factory_with_node_types( chain_spec: Arc, +) -> ProviderFactory>>> { + create_test_provider_factory_with_node_types_and_genesis(chain_spec, 0) +} + +fn create_test_provider_factory_with_node_types_and_genesis( + chain_spec: Arc, + genesis_block_number: u64, ) -> ProviderFactory>>> { // Create a single temp directory that contains all data dirs (db, static_files, rocksdb). // TempDatabase will clean up the entire directory on drop. @@ -72,7 +96,10 @@ pub fn create_test_provider_factory_with_node_types( ProviderFactory::new( db, chain_spec, - StaticFileProvider::read_write(static_files_path).expect("static file provider"), + StaticFileProviderBuilder::read_write(static_files_path) + .with_genesis_block_number(genesis_block_number) + .build() + .expect("static file provider"), RocksDBBuilder::new(&rocksdb_path) .with_default_tables() .build() diff --git a/docs/vocs/docs/pages/cli/reth/import-era.mdx b/docs/vocs/docs/pages/cli/reth/import-era.mdx index 06929db914b..09dcc54ee37 100644 --- a/docs/vocs/docs/pages/cli/reth/import-era.mdx +++ b/docs/vocs/docs/pages/cli/reth/import-era.mdx @@ -148,6 +148,14 @@ Storage: The file containing the block is imported up to and including this height, then the import ends. By default all available blocks are imported. + --with-receipts + Backfill the `Receipts` static file segment from its tip up to the Execution checkpoint. + + Only a missing tail is detected and repaired, never a gap inside the existing segment. The + backfill must reach the checkpoint exactly, so this cannot bootstrap a fresh database: + above the checkpoint receipts are pruned on the next node start, below it the node cannot + start. Byzantium onwards only, from `.era1` or `.ere` files that carry receipts. + Logging: --log.stdout.format The format to use for logs written to stdout From 3aa62806d917023afcca9f4ebbaf06a7ceefe6f2 Mon Sep 17 00:00:00 2001 From: Joao Sousa Date: Fri, 31 Jul 2026 12:01:20 +0100 Subject: [PATCH 073/148] perf(rpc/ipc): O(N) `StreamCodec` via state-carry across `decode()` calls (#26540) Co-authored-by: Matthias Seitz --- crates/rpc/ipc/src/stream_codec.rs | 374 +++++++++++++++++++++++++++-- 1 file changed, 349 insertions(+), 25 deletions(-) diff --git a/crates/rpc/ipc/src/stream_codec.rs b/crates/rpc/ipc/src/stream_codec.rs index be54eaa0a1d..f01a455a93c 100644 --- a/crates/rpc/ipc/src/stream_codec.rs +++ b/crates/rpc/ipc/src/stream_codec.rs @@ -52,6 +52,7 @@ impl Default for Separator { pub struct StreamCodec { incoming_separator: Separator, outgoing_separator: Separator, + scan: ScanState, } impl StreamCodec { @@ -62,7 +63,34 @@ impl StreamCodec { /// New custom stream codec pub const fn new(incoming_separator: Separator, outgoing_separator: Separator) -> Self { - Self { incoming_separator, outgoing_separator } + Self { incoming_separator, outgoing_separator, scan: ScanState::new() } + } +} + +/// Scan state of the [`Separator::Empty`] decoder, carried across `decode` calls so bytes +/// scanned by a previous call are not scanned again. +/// +/// Indices refer to the current read buffer and must be reset whenever bytes are consumed from +/// it. +#[derive(Debug)] +struct ScanState { + depth: i32, + in_str: bool, + is_escaped: bool, + start_idx: usize, + whitespaces: usize, + cursor: usize, +} + +impl ScanState { + const fn new() -> Self { + Self { depth: 0, in_str: false, is_escaped: false, start_idx: 0, whitespaces: 0, cursor: 0 } + } +} + +impl Default for ScanState { + fn default() -> Self { + Self::new() } } @@ -89,36 +117,41 @@ impl tokio_util::codec::Decoder for StreamCodec { Ok(None) } } else { - let mut depth = 0; - let mut in_str = false; - let mut is_escaped = false; - let mut start_idx = 0; - let mut whitespaces = 0; - - for idx in 0..buf.as_ref().len() { - let byte = buf.as_ref()[idx]; - - if (byte == b'{' || byte == b'[') && !in_str { - if depth == 0 { - start_idx = idx; + // resume scanning at the byte the previous call stopped at + while self.scan.cursor < buf.len() { + let idx = self.scan.cursor; + let byte = buf[idx]; + + if (byte == b'{' || byte == b'[') && !self.scan.in_str { + if self.scan.depth == 0 { + self.scan.start_idx = idx; } - depth += 1; - } else if (byte == b'}' || byte == b']') && !in_str { - depth -= 1; - } else if byte == b'"' && !is_escaped { - in_str = !in_str; + self.scan.depth += 1; + } else if (byte == b'}' || byte == b']') && !self.scan.in_str { + self.scan.depth -= 1; + } else if byte == b'"' && !self.scan.is_escaped { + self.scan.in_str = !self.scan.in_str; } else if is_whitespace(byte) { - whitespaces += 1; + self.scan.whitespaces += 1; } - is_escaped = byte == b'\\' && !is_escaped && in_str; - - if depth == 0 && idx != start_idx && idx - start_idx + 1 > whitespaces { - if start_idx > 0 { - buf.advance(start_idx); + self.scan.is_escaped = byte == b'\\' && !self.scan.is_escaped && self.scan.in_str; + + if self.scan.depth == 0 && + idx != self.scan.start_idx && + idx - self.scan.start_idx + 1 > self.scan.whitespaces + { + let start = self.scan.start_idx; + let end = idx + 1; + // reset before advancing the buffer because the stored indices go stale + self.scan = ScanState::new(); + if start > 0 { + buf.advance(start); } - let bts = buf.split_to(idx + 1 - start_idx); + let bts = buf.split_to(end - start); return Ok(String::from_utf8(bts.into()).ok()) } + + self.scan.cursor += 1; } Ok(None) } @@ -142,6 +175,7 @@ impl tokio_util::codec::Encoder for StreamCodec { mod tests { use super::*; use bytes::BufMut; + use rand::{rngs::StdRng, RngCore, SeedableRng}; use tokio_util::codec::Decoder; #[test] @@ -312,4 +346,294 @@ mod tests { assert!(parsed.is_ok(), "serde_json should accept whitespace-wrapped JSON"); assert_eq!(parsed.unwrap(), Obj { key: "value".into() }); } + + /// Multiple messages fed one byte at a time. + #[test] + fn pipelined_messages_byte_by_byte() { + let payload = br#"{"a":1}{"b":2}{"c":3}"#; + let mut buf = BytesMut::with_capacity(256); + let mut codec = StreamCodec::stream_incoming(); + let mut decoded = Vec::new(); + + for byte in payload { + buf.put_u8(*byte); + while let Some(m) = codec.decode(&mut buf).unwrap() { + decoded.push(m); + } + } + + assert_eq!(decoded, vec![r#"{"a":1}"#, r#"{"b":2}"#, r#"{"c":3}"#]); + assert!(buf.is_empty()); + } + + /// Escape sequence split across two `decode` calls, `is_escaped` must carry over. + #[test] + fn escape_split_across_chunk_boundary() { + let mut codec = StreamCodec::stream_incoming(); + let mut buf = BytesMut::with_capacity(64); + + buf.put_slice(br#"{"a":"\"#); + assert!(codec.decode(&mut buf).unwrap().is_none()); + + buf.put_slice(br#"\"}"#); + let msg = codec.decode(&mut buf).unwrap().unwrap(); + assert_eq!(msg, r#"{"a":"\\"}"#); + } + + /// Opening and closing bracket arriving in separate `decode` calls. + #[test] + fn depth_split_across_chunk_boundary() { + let mut codec = StreamCodec::stream_incoming(); + let mut buf = BytesMut::with_capacity(64); + + buf.put_u8(b'{'); + assert!(codec.decode(&mut buf).unwrap().is_none()); + buf.put_u8(b'}'); + assert_eq!(codec.decode(&mut buf).unwrap().unwrap(), "{}"); + } + + /// Input that drives the depth negative must not wedge the codec. + #[test] + fn leading_close_bracket_does_not_poison() { + let mut codec = StreamCodec::stream_incoming(); + let mut buf = BytesMut::with_capacity(8); + buf.put_slice(b"]{"); + assert_eq!(codec.decode(&mut buf).unwrap(), Some("]{".to_string())); + } + + /// `Separator::Byte` decoding fed one byte at a time. + #[test] + fn byte_separator_split_at_every_byte() { + let payload = b"first line\nsecond line\nthird line\n"; + let mut codec = StreamCodec::default(); + let mut buf = BytesMut::with_capacity(64); + let mut got = Vec::new(); + + for byte in payload { + buf.put_u8(*byte); + while let Some(m) = codec.decode(&mut buf).unwrap() { + got.push(m); + } + } + + assert_eq!(got, vec!["first line", "second line", "third line"]); + } + + /// The stateless `Separator::Empty` decode loop that `ScanState` replaced, kept as the + /// reference implementation for the differential tests below. + struct ReferenceDecoder; + + impl Decoder for ReferenceDecoder { + type Item = String; + type Error = io::Error; + + fn decode(&mut self, buf: &mut BytesMut) -> io::Result> { + let mut depth = 0i32; + let mut in_str = false; + let mut is_escaped = false; + let mut start_idx = 0usize; + let mut whitespaces = 0usize; + + for idx in 0..buf.as_ref().len() { + let byte = buf.as_ref()[idx]; + + if (byte == b'{' || byte == b'[') && !in_str { + if depth == 0 { + start_idx = idx; + } + depth += 1; + } else if (byte == b'}' || byte == b']') && !in_str { + depth -= 1; + } else if byte == b'"' && !is_escaped { + in_str = !in_str; + } else if is_whitespace(byte) { + whitespaces += 1; + } + is_escaped = byte == b'\\' && !is_escaped && in_str; + + if depth == 0 && idx != start_idx && idx - start_idx + 1 > whitespaces { + if start_idx > 0 { + buf.advance(start_idx); + } + let bts = buf.split_to(idx + 1 - start_idx); + return Ok(String::from_utf8(bts.into()).ok()) + } + } + Ok(None) + } + } + + /// Feeds the same bytes with the same chunk boundaries to the codec and the reference + /// decoder and returns the decode results of both. Asserts that both consume the same bytes. + fn drain_both(bytes: &[u8], chunks: &[usize]) -> (Vec>, Vec>) { + let mut ours = StreamCodec::stream_incoming(); + let mut theirs = ReferenceDecoder; + + let mut buf_ours = BytesMut::with_capacity(bytes.len() + 16); + let mut buf_theirs = BytesMut::with_capacity(bytes.len() + 16); + let mut out_ours = Vec::new(); + let mut out_theirs = Vec::new(); + + let mut cur = 0usize; + for &c in chunks { + let end = cur.saturating_add(c.max(1)).min(bytes.len()); + if cur == end { + break; + } + buf_ours.put_slice(&bytes[cur..end]); + buf_theirs.put_slice(&bytes[cur..end]); + loop { + let a = ours.decode(&mut buf_ours).unwrap(); + let b = theirs.decode(&mut buf_theirs).unwrap(); + out_ours.push(a.clone()); + out_theirs.push(b.clone()); + if a.is_none() && b.is_none() { + break; + } + } + cur = end; + } + if cur < bytes.len() { + buf_ours.put_slice(&bytes[cur..]); + buf_theirs.put_slice(&bytes[cur..]); + loop { + let a = ours.decode(&mut buf_ours).unwrap(); + let b = theirs.decode(&mut buf_theirs).unwrap(); + out_ours.push(a.clone()); + out_theirs.push(b.clone()); + if a.is_none() && b.is_none() { + break; + } + } + } + + assert_eq!(buf_ours, buf_theirs, "residual buffers diverged"); + + (out_ours, out_theirs) + } + + /// Hand-picked adversarial inputs under several chunk schedules. + #[test] + fn differential_curated_corpus() { + let inputs: &[&[u8]] = &[ + b"", + b"{}", + b"[]", + b"{}{}", + br#"{"a":1}"#, + br#"{"a":1}{"b":2}"#, + b" {} ", + b" { } ", + b" {\n}\n ", + br#"{"a":"\""}"#, + br#"{"a":"\\"}"#, + br#"{"a":"\\\""}"#, + b"{\xff}", + b"{\xff}{\"a\":1}", + b"]{}", + b"}{}", + b"]{}{}", + b"}{}{}", + b"][{}", + b"abc", + b"a", + b"\n\n\n", + br#"{"\":1}"#, + br#"{"escaped":"line\nline"}"#, + b"{[]}", + b"[{}]", + b"[{},{},{}]", + ]; + let chunk_variants: &[&[usize]] = + &[&[usize::MAX], &[1], &[2], &[3], &[5], &[7], &[1, 3, 2, 7, 1, 4]]; + + for input in inputs { + for chunks in chunk_variants { + let (ours, theirs) = drain_both(input, chunks); + assert_eq!( + ours, + theirs, + "differential mismatch on {:?} with chunks {:?}", + std::str::from_utf8(input).unwrap_or(""), + chunks + ); + } + } + } + + /// Random byte sequences over a JSON-heavy alphabet under random chunk schedules. + #[test] + fn differential_random_bytes() { + const SEQUENCES: usize = 200; + const SCHEDULES: usize = 5; + const MAX_LEN: usize = 128; + + let mut rng = StdRng::seed_from_u64(0x00CC_2259_C0DE_C0DE); + let alphabet: &[u8] = b"{}[]\":,\\ \n \t a1{}[]\""; + + for seq in 0..SEQUENCES { + let len = 1 + (rng.next_u32() as usize % MAX_LEN); + let mut bytes = Vec::with_capacity(len); + for _ in 0..len { + let b = alphabet[rng.next_u32() as usize % alphabet.len()]; + bytes.push(b); + } + for sched in 0..SCHEDULES { + let mut chunks = Vec::new(); + let mut remaining = len; + while remaining > 0 { + let c = 1 + (rng.next_u32() as usize % remaining.min(16)); + chunks.push(c); + remaining -= c; + } + let (ours, theirs) = drain_both(&bytes, &chunks); + assert_eq!( + ours, theirs, + "differential mismatch on seq={} sched={} bytes={:?}", + seq, sched, bytes + ); + } + } + } + + /// Randomly chunked well-formed JSON-RPC envelopes. + #[test] + fn differential_random_wellformed_json_rpc() { + let mut rng = StdRng::seed_from_u64(0x00DE_C0DE_C002_2259); + for _ in 0..20 { + let str_len = 500 + (rng.next_u32() as usize % 4096); + let filler: String = (0..str_len) + .map(|_| { + let b = rng.next_u32() as u8 & 0x7F; + // printable ASCII without quotes or backslashes + if b < 0x20 || b == b'"' || b == b'\\' { + b'a' + } else { + b + } + }) + .map(|b| b as char) + .collect(); + let msg = format!( + r#"{{"jsonrpc":"2.0","method":"engine_test","params":["{}"],"id":1}}"#, + filler + ); + let bytes = msg.as_bytes(); + + let mut chunks = Vec::new(); + let mut remaining = bytes.len(); + while remaining > 0 { + let c = 1 + (rng.next_u32() as usize % remaining.min(64)); + chunks.push(c); + remaining -= c; + } + + let (ours, theirs) = drain_both(bytes, &chunks); + assert_eq!(ours, theirs, "differential mismatch on well-formed JSON-RPC"); + assert!( + ours.iter().any(|o| o.is_some()), + "well-formed message should decode to at least one frame" + ); + } + } } From d2ede2c3a8428a3cf816157c687c57599fe168a2 Mon Sep 17 00:00:00 2001 From: rakita Date: Fri, 31 Jul 2026 15:55:29 +0200 Subject: [PATCH 074/148] chore: bump revm to 42.0.1 (#26490) --- .github/scripts/hive/build_simulators.sh | 6 +- Cargo.lock | 343 ++++++++++-------- Cargo.toml | 16 +- .../engine/tree/src/tree/precompile_cache.rs | 4 + crates/transaction-pool/src/validate/eth.rs | 111 +++++- 5 files changed, 319 insertions(+), 161 deletions(-) diff --git a/.github/scripts/hive/build_simulators.sh b/.github/scripts/hive/build_simulators.sh index c87b3be58d9..8f3b320403d 100755 --- a/.github/scripts/hive/build_simulators.sh +++ b/.github/scripts/hive/build_simulators.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash set -eo pipefail -fixture_variant="${1:-osaka}" +fixture_variant="${1:-amsterdam}" case "${fixture_variant}" in amsterdam) - eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/bal@v7.1.1/fixtures_bal.tar.gz" - eels_branch="devnets/bal/7" + eels_fixtures="https://github.com/ethereum/execution-specs/releases/download/tests-glamsterdam-devnet@v7.2.1/fixtures_glamsterdam-devnet.tar.gz" + eels_branch="devnets/glamsterdam/7" ;; osaka) eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz" diff --git a/Cargo.lock b/Cargo.lock index 971d49f1190..8a217855dbe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,14 +286,14 @@ dependencies = [ "ethereum_ssz_derive", "serde", "serde_with", - "sha2", + "sha2 0.10.9", ] [[package]] name = "alloy-evm" -version = "0.37.1" +version = "0.38.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "acde665074b478c97047dc2a461b4916cac77922d690e5b7415d1941ae7ff7bf" +checksum = "6b307a52f91f7f3aad56b5cb4f54e5eb17dd0b3f4b2c82823301499fe1c6fdaf" dependencies = [ "alloy-consensus", "alloy-eips", @@ -1031,7 +1031,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1042,7 +1042,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1091,43 +1091,43 @@ checksum = "70e0a5f99dfebb87bb342d0f53bb92c81842e100bbb915223e38349580e5441d" [[package]] name = "ark-bls12-381" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3df4dcc01ff89867cd86b0da835f23c3f02738353aaee7dde7495af71363b8d5" +checksum = "be2ede2c0c96fa37d5d3484e8a59fec566c4a52b8c84bf993eaa6c67d7225a4c" dependencies = [ "ark-ec", - "ark-ff 0.5.0", - "ark-serialize 0.5.0", - "ark-std 0.5.0", + "ark-ff 0.6.0", + "ark-serialize 0.6.0", + "ark-std 0.6.0", ] [[package]] name = "ark-bn254" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d69eab57e8d2663efa5c63135b2af4f396d66424f88954c21104125ab6b3e6bc" +checksum = "6bc66f96ebe2a17a499475b4f94791d379817592ef494171586967ffdc6f95db" dependencies = [ "ark-ec", - "ark-ff 0.5.0", + "ark-ff 0.6.0", "ark-r1cs-std", - "ark-std 0.5.0", + "ark-std 0.6.0", ] [[package]] name = "ark-ec" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d68f2d516162846c1238e755a7c4d131b892b70cc70c471a8e3ca3ed818fce" +checksum = "8352a2b2aedf6ba2cc38f7520fc51191d518dde96175c729af19f2d059f191c4" dependencies = [ "ahash", - "ark-ff 0.5.0", + "ark-ff 0.6.0", "ark-poly", - "ark-serialize 0.5.0", - "ark-std 0.5.0", + "ark-serialize 0.6.0", + "ark-std 0.6.0", "educe", "fnv", - "hashbrown 0.15.5", - "itertools 0.13.0", + "hashbrown 0.17.0", + "itertools 0.14.0", "num-bigint", "num-integer", "num-traits", @@ -1302,30 +1302,31 @@ dependencies = [ [[package]] name = "ark-poly" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "579305839da207f02b89cd1679e50e67b4331e2f9294a57693e5051b7703fe27" +checksum = "75f55af10b672002b8d953e230282c51206842e20e5791a94432219b4201de5c" dependencies = [ "ahash", - "ark-ff 0.5.0", - "ark-serialize 0.5.0", - "ark-std 0.5.0", + "ark-ff 0.6.0", + "ark-serialize 0.6.0", + "ark-std 0.6.0", "educe", "fnv", - "hashbrown 0.15.5", + "hashbrown 0.17.0", ] [[package]] name = "ark-r1cs-std" -version = "0.5.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "941551ef1df4c7a401de7068758db6503598e6f01850bdb2cfdb614a1f9dbea1" +checksum = "291f1c6628bfcac79b0dc2adbe401aa9100e2e96daa971645e0b18fc94de9a98" dependencies = [ "ark-ec", - "ark-ff 0.5.0", + "ark-ff 0.6.0", "ark-relations", - "ark-std 0.5.0", + "ark-std 0.6.0", "educe", + "itertools 0.14.0", "num-bigint", "num-integer", "num-traits", @@ -1334,14 +1335,18 @@ dependencies = [ [[package]] name = "ark-relations" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec46ddc93e7af44bcab5230937635b06fb5744464dd6a7e7b083e80ebd274384" +checksum = "fe4c11c797a64b8a23e22bf4e77bf582ac27bb21395e3183a9a506ba2561e9f9" dependencies = [ - "ark-ff 0.5.0", - "ark-std 0.5.0", + "ark-ff 0.6.0", + "ark-poly", + "ark-serialize 0.6.0", + "ark-std 0.6.0", + "foldhash 0.1.5", + "indexmap 2.14.0", "tracing", - "tracing-subscriber 0.2.25", + "tracing-subscriber", ] [[package]] @@ -1371,7 +1376,6 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f4d068aaf107ebcd7dfb52bc748f8030e0fc930ac8e360146ca54c1203088f7" dependencies = [ - "ark-serialize-derive 0.5.0", "ark-std 0.5.0", "arrayvec", "digest 0.10.7", @@ -1384,24 +1388,13 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a74dd304fd536fb95d0a328e72be759209cc496a9da094c5bc56e5fea4f9e86b" dependencies = [ - "ark-serialize-derive 0.6.0", + "ark-serialize-derive", "ark-std 0.6.0", "digest 0.10.7", "num-bigint", "serde_with", ] -[[package]] -name = "ark-serialize-derive" -version = "0.5.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "213888f660fddcca0d257e88e54ac05bca01885f258ccdf695bafd77031bb69d" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.118", -] - [[package]] name = "ark-serialize-derive" version = "0.6.0" @@ -1905,7 +1898,7 @@ dependencies = [ "hashbrown 0.16.1", "indexmap 2.14.0", "once_cell", - "phf", + "phf 0.13.1", "rustc-hash", "static_assertions", ] @@ -1972,7 +1965,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -2041,7 +2034,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf88ba1141d185c399bee5288d850d63b8369520c1eafc32a0430b5b6c287bf4" dependencies = [ - "sha2", + "sha2 0.10.9", "tinyvec", ] @@ -2422,7 +2415,7 @@ dependencies = [ "hmac", "k256", "serde", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", ] @@ -2438,7 +2431,7 @@ dependencies = [ "once_cell", "pbkdf2", "rand 0.8.6", - "sha2", + "sha2 0.10.9", "thiserror 1.0.69", ] @@ -2454,9 +2447,9 @@ dependencies = [ "const-hex", "digest 0.10.7", "generic-array", - "ripemd", + "ripemd 0.1.3", "serde", - "sha2", + "sha2 0.10.9", "sha3 0.10.8", "thiserror 1.0.69", ] @@ -3257,7 +3250,7 @@ dependencies = [ "ed25519", "rand_core 0.6.4", "serde", - "sha2", + "sha2 0.10.9", "subtle", "zeroize", ] @@ -3434,7 +3427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3445,7 +3438,7 @@ checksum = "5aa93f58bb1eb3d1e556e4f408ef1dac130bad01ac37db4e7ade45de40d1c86a" dependencies = [ "cpufeatures 0.2.17", "ring", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -4453,7 +4446,6 @@ version = "0.15.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" dependencies = [ - "allocator-api2", "foldhash 0.1.5", ] @@ -5179,7 +5171,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -5534,7 +5526,7 @@ dependencies = [ "elliptic-curve", "once_cell", "serdect", - "sha2", + "sha2 0.10.9", "signature", ] @@ -5693,7 +5685,7 @@ dependencies = [ "k256", "multihash", "quick-protobuf", - "sha2", + "sha2 0.10.9", "thiserror 2.0.18", "tracing", "zeroize", @@ -5833,7 +5825,7 @@ dependencies = [ "generator", "scoped-tls", "tracing", - "tracing-subscriber 0.3.23", + "tracing-subscriber", ] [[package]] @@ -6256,7 +6248,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -6490,7 +6482,7 @@ dependencies = [ "opentelemetry", "tracing", "tracing-core", - "tracing-subscriber 0.3.23", + "tracing-subscriber", ] [[package]] @@ -6587,7 +6579,7 @@ dependencies = [ "ecdsa", "elliptic-curve", "primeorder", - "sha2", + "sha2 0.10.9", ] [[package]] @@ -6746,8 +6738,18 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_macros", - "phf_shared", + "phf_macros 0.13.1", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "010378780309880b08997fae13be7834dba947d36393bd372f2b1556deb2a2f6" +dependencies = [ + "phf_macros 0.14.0", + "phf_shared 0.14.0", "serde", ] @@ -6758,7 +6760,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" dependencies = [ "fastrand", - "phf_shared", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_generator" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aeb62e0959d5a1bebc965f4d15d9e2b7cea002b6b0f5ba8cde6cc26738467100" +dependencies = [ + "fastrand", + "phf_shared 0.14.0", ] [[package]] @@ -6767,8 +6779,21 @@ version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn 2.0.118", +] + +[[package]] +name = "phf_macros" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5fa8d0ca26d424d27630da600c6624696e7dec8bf7b3b492b383c5dc49e5e085" +dependencies = [ + "phf_generator 0.14.0", + "phf_shared 0.14.0", "proc-macro2", "quote", "syn 2.0.118", @@ -6783,6 +6808,15 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fd9027e2d9319be6349febd1db4e8d02aa544921200c9b777720ac34a3aa89" +dependencies = [ + "siphasher", +] + [[package]] name = "pin-project" version = "1.1.13" @@ -7934,9 +7968,9 @@ dependencies = [ [[package]] name = "reth-codecs" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb43f4858fef4e3b42b80954d2edf1ff67c8ad7498347083cdc7d0f6eff2f081" +checksum = "1f0dd7f542fbcae0e490ebe13f3eababcc05547375246ff7f62700b533cf3f01" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7955,9 +7989,9 @@ dependencies = [ [[package]] name = "reth-codecs-derive" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5798ae4d6b264764bc0d742468bd32c7fb515e774645d4930f95ff6311f3ae14" +checksum = "e7e508fcbb8cb775639dbbb56b81b62756d4322f28e76c60a52aa32575f993fe" dependencies = [ "proc-macro2", "quote", @@ -8335,7 +8369,7 @@ dependencies = [ "rand 0.8.6", "reth-network-peers", "secp256k1 0.30.0", - "sha2", + "sha2 0.10.9", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -8507,7 +8541,7 @@ dependencies = [ "reqwest", "reth-era-downloader", "reth-ethereum-primitives", - "sha2", + "sha2 0.10.9", "snap", "tempfile", "test-case", @@ -8527,7 +8561,7 @@ dependencies = [ "reqwest", "reth-era", "reth-fs-util", - "sha2", + "sha2 0.10.9", "tempfile", "test-case", "tokio", @@ -9647,7 +9681,7 @@ dependencies = [ "reth-trie-common", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror 2.0.18", "tokio", ] @@ -9672,9 +9706,9 @@ dependencies = [ [[package]] name = "reth-primitives-traits" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f59c928b2865af5bcdbce94800270b7f7798f27a22ca0aabdd2b7d3e6587c9ce" +checksum = "4666a8d5b246650b23e6767a45e46334d352557f473e91c2ea7211cc29291cca" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9896,7 +9930,7 @@ dependencies = [ "revm-inspectors", "serde", "serde_json", - "sha2", + "sha2 0.10.9", "thiserror 2.0.18", "tokio", "tokio-stream", @@ -10200,9 +10234,9 @@ dependencies = [ [[package]] name = "reth-rpc-traits" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace77dbcdd59c014fda4565ebfec76cd1fe6f5d98584b4655e8d22a947b9eee3" +checksum = "b417a50d3fa9b58bb1a22e46fb47e0866006f24abffa1fbe75ae60116b5b2efa" dependencies = [ "alloy-consensus", "alloy-network", @@ -10525,7 +10559,7 @@ dependencies = [ "tracing-journald", "tracing-logfmt", "tracing-samply", - "tracing-subscriber 0.3.23", + "tracing-subscriber", "tracing-tracy", "tracy-client", ] @@ -10544,7 +10578,7 @@ dependencies = [ "opentelemetry_sdk", "tracing", "tracing-opentelemetry", - "tracing-subscriber 0.3.23", + "tracing-subscriber", "url", ] @@ -10751,18 +10785,18 @@ dependencies = [ [[package]] name = "reth-zstd-compressors" -version = "0.5.2" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1bd4ae4f54a2ba813eef082910bc646bd31cab8ed134308fa71f1068331fb84" +checksum = "80addacffc5df9398c55919f8993922d2b688f2aa5753708fdd658ca3cc63e39" dependencies = [ "zstd", ] [[package]] name = "revm" -version = "41.0.0" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d51c254839fb36357d0b02d02e46ba57e683d7b017e2da33b47ae74685a9e59" +checksum = "b7c7fcd426beab3b91300a6d0d5eb8d583c0fad0fba451126e19fd51443eca62" dependencies = [ "revm-bytecode", "revm-context", @@ -10779,22 +10813,22 @@ dependencies = [ [[package]] name = "revm-bytecode" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e33493cf6d996e9242db4f2002caef48ec9aa8c4f3fff3b18aed10ef3942eec7" +checksum = "537356bdec80dbc04e4e6bb688b3441ff7cf18eecdcf3cac23150633c46467f1" dependencies = [ "bitvec", "paste", - "phf", + "phf 0.14.0", "revm-primitives", "serde", ] [[package]] name = "revm-context" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86e48fc736d9542c082ea5305be44b6a14b53a615f0147ad57f62189fd813068" +checksum = "cc6512f05f83378e29b2b016fa5bd7f5a533e3625548be4113732e1a536c2564" dependencies = [ "bitvec", "cfg-if", @@ -10809,9 +10843,9 @@ dependencies = [ [[package]] name = "revm-context-interface" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac68282a454318246817486835a446519291dd0a8167d09de14c7e96f2147696" +checksum = "103fff818b10311034c0c11ff8760294ee0e3c320daab3c5717990fe8ee2617b" dependencies = [ "alloy-eip2930", "alloy-eip7702", @@ -10825,9 +10859,9 @@ dependencies = [ [[package]] name = "revm-database" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d761681a43e48408ea23f7f66ff56ff7b6128cadb8f9135d1b94787d0c0fc6b4" +checksum = "df72760eab0852100e748c7cb03ce0221626d60ac14534118c84c5cc894fb4c9" dependencies = [ "alloy-eips", "derive_more", @@ -10841,9 +10875,9 @@ dependencies = [ [[package]] name = "revm-database-interface" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b96e37d62fa60b45987b408486c84ebef4af4e7ddf1b3b96b4955a7263fd4e92" +checksum = "d9286c059a11be1dee814cafdc3a3ade57521c8caa050785a52c19a9d894f400" dependencies = [ "auto_impl", "either", @@ -10855,9 +10889,9 @@ dependencies = [ [[package]] name = "revm-handler" -version = "41.0.0" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f5e043ebfc0899c435e44475796285898e4f822e637bd67856da79cc170bd9df" +checksum = "09126175c3fa482e71e80cc28c6bb17e681911e472b94d4f7f73dd6780411798" dependencies = [ "auto_impl", "derive-where", @@ -10874,9 +10908,9 @@ dependencies = [ [[package]] name = "revm-inspector" -version = "41.0.0" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e470b2a5e177918944d8cd26174455b61b8fbfbbe7fe411d28b4097e410d11e" +checksum = "4aa55b208b52f0c083ea68147c78866eeecadd7c0071a614f477de316b60648e" dependencies = [ "auto_impl", "either", @@ -10892,9 +10926,9 @@ dependencies = [ [[package]] name = "revm-inspectors" -version = "0.41.2" +version = "0.42.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "54e22e1aa328f78e8c3dea6a3a860a3a3c3a77c4a7e775e3fdd3dcbb24a152ac" +checksum = "3958b5de5c2c08c6a49b8d94f6ead3e9cd7980c43d6ed240aa3d6485d53269b5" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -10912,9 +10946,9 @@ dependencies = [ [[package]] name = "revm-interpreter" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa5547f653f9e4c47b4f9a9075a0b7c0faea5fa29b7873f392733a943837825d" +checksum = "fd310927437752ce4bac6997b385772823b84843f04c82e151993bec1265a5ed" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -10925,15 +10959,15 @@ dependencies = [ [[package]] name = "revm-precompile" -version = "41.0.0" +version = "42.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d785931e4b8750cf9d79c808c22f05979c8d191baafc8badd4651db82584c1c" +checksum = "f2431091420999795df6a200b9f7933defe3d654710560a895e6bcabbd9954a5" dependencies = [ "ark-bls12-381", "ark-bn254", "ark-ec", - "ark-ff 0.5.0", - "ark-serialize 0.5.0", + "ark-ff 0.6.0", + "ark-serialize 0.6.0", "arrayref", "aurora-engine-modexp", "aws-lc-rs", @@ -10945,16 +10979,16 @@ dependencies = [ "p256", "revm-context-interface", "revm-primitives", - "ripemd", + "ripemd 0.2.0", "secp256k1 0.31.1", - "sha2", + "sha2 0.11.0", ] [[package]] name = "revm-primitives" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66f39151a31afe93d97f7ac894dd9dcc989368d5ccba19ce079bfc1d73b77452" +checksum = "2ef5564c1bce219fd08beca43d4f39fdcbc19193fa4f604c0c6b58950a13194a" dependencies = [ "alloy-primitives", "once_cell", @@ -10963,9 +10997,9 @@ dependencies = [ [[package]] name = "revm-state" -version = "41.0.0" +version = "42.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73ef2fffc28acc41e8ef7525f2b1d16288a10f80579a1b33e7e62abc9892314a" +checksum = "2c4ae66db3abfcb565854dbfc95513bfa440b3dc390ddf8e50b0363d12a6fa79" dependencies = [ "alloy-eip7928", "bitflags 2.13.0", @@ -10978,7 +11012,7 @@ dependencies = [ [[package]] name = "revmc" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "revm-bytecode", "revm-context-interface", @@ -10998,7 +11032,7 @@ dependencies = [ [[package]] name = "revmc-backend" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "eyre", "ruint", @@ -11007,12 +11041,12 @@ dependencies = [ [[package]] name = "revmc-build" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" [[package]] name = "revmc-builtins" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "paste", "revm-bytecode", @@ -11027,7 +11061,7 @@ dependencies = [ [[package]] name = "revmc-codegen" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "alloy-primitives", "bitflags 2.13.0", @@ -11058,7 +11092,7 @@ dependencies = [ [[package]] name = "revmc-context" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "revm-context", "revm-context-interface", @@ -11073,7 +11107,7 @@ dependencies = [ [[package]] name = "revmc-llvm" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "alloy-primitives", "cc", @@ -11086,7 +11120,7 @@ dependencies = [ [[package]] name = "revmc-runtime" version = "0.1.0" -source = "git+https://github.com/paradigmxyz/revmc?branch=main#520462a463523a3bcd0a47226ddbc3200d62232e" +source = "git+https://github.com/paradigmxyz/revmc?branch=main#cf68a87f627299a9c49bcc333a8a317c9b312a3d" dependencies = [ "alloy-evm", "alloy-primitives", @@ -11159,6 +11193,15 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "ripemd" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4dd4211456b4172d7e44261920c25acf07367c4f04bb5f5d54fc21b090d9b159" +dependencies = [ + "digest 0.11.3", +] + [[package]] name = "rlimit" version = "0.11.0" @@ -11326,7 +11369,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11406,7 +11449,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -11803,6 +11846,17 @@ dependencies = [ "digest 0.10.7", ] +[[package]] +name = "sha2" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "446ba717509524cb3f22f17ecc096f10f4822d76ab5c0b9822c5f9c284e825f4" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.3", +] + [[package]] name = "sha3" version = "0.10.8" @@ -12028,7 +12082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -12276,7 +12330,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -12816,7 +12870,7 @@ dependencies = [ "symlink", "thiserror 2.0.18", "time", - "tracing-subscriber 0.3.23", + "tracing-subscriber", ] [[package]] @@ -12838,7 +12892,7 @@ checksum = "bf0a738ed5d6450a9fb96e86a23ad808de2b727fd1394585da5cdd6788ffe724" dependencies = [ "serde_json", "tracing-core", - "tracing-subscriber 0.3.23", + "tracing-subscriber", ] [[package]] @@ -12869,7 +12923,7 @@ checksum = "2d3a81ed245bfb62592b1e2bc153e77656d94ee6a0497683a65a12ccaf2438d0" dependencies = [ "libc", "tracing-core", - "tracing-subscriber 0.3.23", + "tracing-subscriber", ] [[package]] @@ -12892,7 +12946,7 @@ dependencies = [ "time", "tracing", "tracing-core", - "tracing-subscriber 0.3.23", + "tracing-subscriber", ] [[package]] @@ -12907,7 +12961,7 @@ dependencies = [ "tracing", "tracing-core", "tracing-log", - "tracing-subscriber 0.3.23", + "tracing-subscriber", "web-time", ] @@ -12924,7 +12978,7 @@ dependencies = [ "memmap2", "smallvec", "tracing-core", - "tracing-subscriber 0.3.23", + "tracing-subscriber", ] [[package]] @@ -12937,15 +12991,6 @@ dependencies = [ "tracing-core", ] -[[package]] -name = "tracing-subscriber" -version = "0.2.25" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0d2eaa99c3c2e41547cfa109e910a68ea03823cccad4a0525dcbc9b01e8c71" -dependencies = [ - "tracing-core", -] - [[package]] name = "tracing-subscriber" version = "0.3.23" @@ -12974,7 +13019,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eaa1852afa96e0fe9e44caa53dc0bd2d9d05e0f2611ce09f97f8677af56e4ba" dependencies = [ "tracing-core", - "tracing-subscriber 0.3.23", + "tracing-subscriber", "tracy-client", ] @@ -13555,7 +13600,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 708eac1249e..4815b98a73f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -322,8 +322,8 @@ reth-cli = { path = "crates/cli/cli" } reth-cli-commands = { path = "crates/cli/commands" } reth-cli-runner = { path = "crates/cli/runner" } reth-cli-util = { path = "crates/cli/util" } -reth-codecs = { version = "0.5.0", default-features = false } -reth-codecs-derive = "0.5.0" +reth-codecs = { version = "0.6.0", default-features = false } +reth-codecs-derive = "0.6.0" reth-config = { path = "crates/config", default-features = false } reth-consensus = { path = "crates/consensus/consensus", default-features = false } reth-consensus-common = { path = "crates/consensus/common", default-features = false } @@ -391,7 +391,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" } reth-payload-primitives = { path = "crates/payload/primitives" } reth-payload-validator = { path = "crates/payload/validator" } reth-payload-util = { path = "crates/payload/util" } -reth-primitives-traits = { version = "0.5.0", default-features = false } +reth-primitives-traits = { version = "0.6.0", default-features = false } reth-provider = { path = "crates/storage/provider" } reth-prune = { path = "crates/prune/prune" } reth-prune-types = { path = "crates/prune/types", default-features = false } @@ -406,7 +406,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal reth-rpc-layer = { path = "crates/rpc/rpc-layer" } reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" } reth-rpc-convert = { path = "crates/rpc/rpc-convert" } -reth-rpc-traits = { version = "0.5.0", default-features = false } +reth-rpc-traits = { version = "0.6.0", default-features = false } reth-stages = { path = "crates/stages/stages" } reth-stages-api = { path = "crates/stages/api" } reth-stages-types = { path = "crates/stages/types", default-features = false } @@ -426,12 +426,12 @@ reth-trie-common = { path = "crates/trie/common", default-features = false } reth-trie-db = { path = "crates/trie/db" } reth-trie-parallel = { path = "crates/trie/parallel" } reth-trie-sparse = { path = "crates/trie/sparse", default-features = false } -reth-zstd-compressors = { version = "0.5.0", default-features = false } +reth-zstd-compressors = { version = "0.6.0", default-features = false } # revm -revm = { version = "41.0.0", default-features = false } +revm = { version = "42.0.1", default-features = false } revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } -revm-inspectors = "0.41.0" +revm-inspectors = "0.42.0" # eth alloy-dyn-abi = "1.6.1" @@ -441,7 +441,7 @@ alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } alloy-eip7928 = { version = "0.4.5", default-features = false, features = ["rlp"] } -alloy-evm = { version = "0.37.0", default-features = false } +alloy-evm = { version = "0.38.0", default-features = false } alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } diff --git a/crates/engine/tree/src/tree/precompile_cache.rs b/crates/engine/tree/src/tree/precompile_cache.rs index a5f0a771e84..e868cca8485 100644 --- a/crates/engine/tree/src/tree/precompile_cache.rs +++ b/crates/engine/tree/src/tree/precompile_cache.rs @@ -265,6 +265,7 @@ mod tests { status: PrecompileStatus::Success, gas_used: 0, state_gas_used: 0, + state_gas_spilled: 0, reservoir: 0, gas_refunded: 0, bytes: Bytes::default(), @@ -279,6 +280,7 @@ mod tests { status: PrecompileStatus::Success, gas_used: 50, state_gas_used: 0, + state_gas_spilled: 0, reservoir: 0, gas_refunded: 0, bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"), @@ -313,6 +315,7 @@ mod tests { status: PrecompileStatus::Success, gas_used: 5000, state_gas_used: 0, + state_gas_spilled: 0, reservoir: 0, gas_refunded: 0, bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"), @@ -330,6 +333,7 @@ mod tests { status: PrecompileStatus::Success, gas_used: 7000, state_gas_used: 0, + state_gas_spilled: 0, reservoir: 0, gas_refunded: 0, bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"), diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index c8e178ab56f..c7fc15fd641 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -915,6 +915,10 @@ where self.fork_tracker.osaka.store(true, std::sync::atomic::Ordering::Relaxed); } + if self.chain_spec().is_amsterdam_active_at_timestamp(new_tip_block.timestamp()) { + self.fork_tracker.amsterdam.store(true, std::sync::atomic::Ordering::Relaxed); + } + self.fork_tracker .tip_timestamp .store(new_tip_block.timestamp(), std::sync::atomic::Ordering::Relaxed); @@ -1028,6 +1032,8 @@ pub struct EthTransactionValidatorBuilder { prague: bool, /// Fork indicator whether we are in the Osaka hardfork. osaka: bool, + /// Fork indicator whether we are in the Amsterdam hardfork. + amsterdam: bool, /// Timestamp of the tip block. tip_timestamp: u64, /// Max blob count at the block's timestamp. @@ -1119,6 +1125,7 @@ impl EthTransactionValidatorBuilder { cancun: chain_spec.is_cancun_active_at_timestamp(tip.timestamp()), prague: chain_spec.is_prague_active_at_timestamp(tip.timestamp()), osaka: chain_spec.is_osaka_active_at_timestamp(tip.timestamp()), + amsterdam: chain_spec.is_amsterdam_active_at_timestamp(tip.timestamp()), tip_timestamp: tip.timestamp(), @@ -1199,6 +1206,17 @@ impl EthTransactionValidatorBuilder { self } + /// Disables the Amsterdam fork. + pub const fn no_amsterdam(self) -> Self { + self.set_amsterdam(false) + } + + /// Set the Amsterdam fork. + pub const fn set_amsterdam(mut self, amsterdam: bool) -> Self { + self.amsterdam = amsterdam; + self + } + /// Disables the support for EIP-2718 transactions. pub const fn no_eip2718(self) -> Self { self.set_eip2718(false) @@ -1333,6 +1351,7 @@ impl EthTransactionValidatorBuilder { cancun, prague, osaka, + amsterdam, tip_timestamp, eip2718, eip1559, @@ -1359,6 +1378,7 @@ impl EthTransactionValidatorBuilder { cancun: AtomicBool::new(cancun), prague: AtomicBool::new(prague), osaka: AtomicBool::new(osaka), + amsterdam: AtomicBool::new(amsterdam), tip_timestamp: AtomicU64::new(tip_timestamp), max_blob_count: AtomicU64::new(max_blob_count), max_initcode_size: AtomicUsize::new(max_initcode_size), @@ -1423,6 +1443,8 @@ pub struct ForkTracker { pub prague: AtomicBool, /// Tracks if osaka is activated at the block's timestamp. pub osaka: AtomicBool, + /// Tracks if amsterdam is activated at the block's timestamp. + pub amsterdam: AtomicBool, /// Tracks max blob count per transaction at the block's timestamp. pub max_blob_count: AtomicU64, /// Tracks the timestamp of the tip block. @@ -1454,6 +1476,11 @@ impl ForkTracker { self.osaka.load(std::sync::atomic::Ordering::Relaxed) } + /// Returns `true` if Amsterdam fork is activated. + pub fn is_amsterdam_activated(&self) -> bool { + self.amsterdam.load(std::sync::atomic::Ordering::Relaxed) + } + /// Returns the timestamp of the tip block. pub fn tip_timestamp(&self) -> u64 { self.tip_timestamp.load(std::sync::atomic::Ordering::Relaxed) @@ -1473,7 +1500,9 @@ pub fn ensure_intrinsic_gas( fork_tracker: &ForkTracker, ) -> Result<(), InvalidPoolTransactionError> { use revm::primitives::hardfork::SpecId; - let spec_id = if fork_tracker.is_prague_activated() { + let spec_id = if fork_tracker.is_amsterdam_activated() { + SpecId::AMSTERDAM + } else if fork_tracker.is_prague_activated() { SpecId::PRAGUE } else if fork_tracker.is_shanghai_activated() { SpecId::SHANGHAI @@ -1481,6 +1510,16 @@ pub fn ensure_intrinsic_gas( SpecId::MERGE }; + // EIP-2780 replaces the flat intrinsic base cost with a decomposed one that depends on + // `tx.to` and `tx.value`. + let eip2780 = fork_tracker.is_amsterdam_activated().then(|| { + revm::context_interface::cfg::gas_params::Eip2780TxInfo { + value: transaction.value(), + // Self-transfer: a `Call` whose recipient is the sender itself. + is_self_transfer: transaction.kind().to() == Some(&transaction.sender()), + } + }); + let gas = revm::interpreter::gas::calculate_initial_tx_gas( spec_id, transaction.input(), @@ -1491,6 +1530,7 @@ pub fn ensure_intrinsic_gas( .map(|l| l.iter().map(|i| i.storage_keys.len()).sum::()) .unwrap_or_default() as u64, transaction.authorization_list().map(|l| l.len()).unwrap_or_default() as u64, + eip2780, ); let gas_limit = transaction.gas_limit(); @@ -1533,6 +1573,74 @@ mod tests { EthPooledTransaction::from_pooled(tx.try_into_recovered().unwrap()) } + fn eip1559_tx( + to: Address, + sender: Address, + value: u64, + gas_limit: u64, + ) -> EthPooledTransaction { + let tx = alloy_consensus::TxEip1559 { + chain_id: 1, + nonce: 0, + gas_limit, + max_fee_per_gas: 1, + max_priority_fee_per_gas: 0, + to: to.into(), + value: U256::from(value), + ..Default::default() + }; + let signed = reth_ethereum_primitives::TransactionSigned::new_unhashed( + tx.into(), + alloy_primitives::Signature::test_signature(), + ); + EthPooledTransaction::new( + alloy_consensus::transaction::Recovered::new_unchecked(signed, sender), + 200, + ) + } + + /// EIP-2780 replaces the flat 21k intrinsic base with a decomposed one: 12k base, plus a cold + /// account access for `tx.to` and a transfer charge when `tx.value` is non-zero, with a + /// carve-out for self-transfers. + #[test] + fn intrinsic_gas_eip2780() { + let sender = Address::repeat_byte(1); + let recipient = Address::repeat_byte(2); + + let amsterdam = || ForkTracker { + shanghai: true.into(), + cancun: true.into(), + prague: true.into(), + osaka: true.into(), + amsterdam: true.into(), + tip_timestamp: 0.into(), + max_blob_count: 0.into(), + max_initcode_size: AtomicUsize::new(MAX_INITCODE_SIZE), + tx_gas_limit_cap: AtomicU64::new(0), + }; + let pre_amsterdam = || ForkTracker { amsterdam: false.into(), ..amsterdam() }; + + // Self-transfer: base cost only (12k), where pre-Amsterdam it pays the flat 21k. + let self_transfer = eip1559_tx(sender, sender, 1, 15_000); + assert!(ensure_intrinsic_gas(&self_transfer, &amsterdam()).is_ok()); + assert!(ensure_intrinsic_gas(&self_transfer, &pre_amsterdam()).is_err()); + + // Zero-value call to another account: base + cold account access (15k). + let zero_value = eip1559_tx(recipient, sender, 0, 15_000); + assert!(ensure_intrinsic_gas(&zero_value, &amsterdam()).is_ok()); + assert!( + ensure_intrinsic_gas(&eip1559_tx(recipient, sender, 0, 14_999), &amsterdam()).is_err() + ); + + // Value transfer to another account: base + cold access + transfer log + value cost (21k). + assert!( + ensure_intrinsic_gas(&eip1559_tx(recipient, sender, 1, 15_000), &amsterdam()).is_err() + ); + assert!( + ensure_intrinsic_gas(&eip1559_tx(recipient, sender, 1, 21_000), &amsterdam()).is_ok() + ); + } + // #[tokio::test] async fn validate_transaction() { @@ -1542,6 +1650,7 @@ mod tests { cancun: false.into(), prague: false.into(), osaka: false.into(), + amsterdam: false.into(), tip_timestamp: 0.into(), max_blob_count: 0.into(), max_initcode_size: AtomicUsize::new(MAX_INITCODE_SIZE), From d043d06c80685ff31380a1e8c7ecba6b1756b8a7 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Fri, 31 Jul 2026 16:10:47 +0200 Subject: [PATCH 075/148] fix(trie): propagate proof worker startup failures (#26545) --- .../tree/src/tree/state_root_strategy/mod.rs | 15 +++- .../tree/state_root_strategy/sparse_trie.rs | 82 +++++++++++++++---- crates/trie/parallel/src/error.rs | 3 + crates/trie/parallel/src/proof_task.rs | 26 +++++- 4 files changed, 102 insertions(+), 24 deletions(-) diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 5de4e56d369..26422dc9666 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -61,7 +61,7 @@ use crate::tree::{ TreeConfig, }; use alloy_primitives::B256; -use crossbeam_channel::Receiver as CrossbeamReceiver; +use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender}; use reth_chain_state::{ExecutedBlock, PreservedSparseTrie}; use reth_errors::ProviderResult; use reth_evm::{ConfigureEvm, OnStateHook}; @@ -79,7 +79,7 @@ use reth_trie::{ hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, }; -use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofWorkerHandle}; +use reth_trie_parallel::proof_task::{ProofResultMessage, ProofTaskCtx, ProofWorkerHandle}; pub use reth_trie_parallel::{ error::StateRootTaskError, state_root_task::{ @@ -539,13 +539,16 @@ impl DefaultStateRootStrategy { } = options; let (updates_tx, from_multi_proof) = crossbeam_channel::unbounded(); let (cancel_guard, cancel_rx) = StateRootTaskCancelGuard::channel(); + let (proof_result_tx, proof_result_rx) = + crossbeam_channel::unbounded::(); let task_ctx = ProofTaskCtx::new(multiproof_provider_factory); #[cfg(feature = "trie-debug")] let task_ctx = task_ctx.with_proof_jitter(config.proof_jitter()); let halve_workers = transaction_count .is_some_and(|count| count <= Self::SMALL_BLOCK_PROOF_WORKER_TX_THRESHOLD); - let proof_handle = ProofWorkerHandle::new(executor, task_ctx, halve_workers); + let proof_handle = + ProofWorkerHandle::new(executor, task_ctx, halve_workers, proof_result_tx.clone()); let (state_root_tx, state_root_rx) = mpsc::channel(); let (hashed_state_tx, hashed_state_rx) = mpsc::channel(); @@ -555,6 +558,8 @@ impl DefaultStateRootStrategy { executor, overlay_manager, proof_handle, + proof_result_tx, + proof_result_rx, state_root_tx, hashed_state_tx, from_multi_proof, @@ -587,6 +592,8 @@ impl DefaultStateRootStrategy { executor: &reth_tasks::Runtime, overlay_manager: &OverlayManager, proof_worker_handle: ProofWorkerHandle, + proof_result_tx: CrossbeamSender, + proof_result_rx: CrossbeamReceiver, state_root_tx: mpsc::Sender>, hashed_state_tx: mpsc::Sender>, from_multi_proof: CrossbeamReceiver, @@ -666,6 +673,8 @@ impl DefaultStateRootStrategy { cancel_rx, hashed_state_tx, proof_worker_handle, + proof_result_tx, + proof_result_rx, trie_metrics.clone(), sparse_state_trie, parent_state_root, diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 982df30d207..2abd518875d 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -22,7 +22,8 @@ use reth_trie_common::{MultiProofTargetsV2, ProofV2Target, ProofV2TargetParent}; use reth_trie_parallel::{ error::StateRootTaskError, proof_task::{ - AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle, + AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofResultSender, + ProofWorkerHandle, }, }; use reth_trie_sparse::{ @@ -35,7 +36,7 @@ use tracing::{debug, debug_span, error, instrument, trace_span}; /// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching. pub(super) struct SparseTrieCacheTask { /// Sender for proof results. - proof_result_tx: CrossbeamSender, + proof_result_tx: ProofResultSender, /// Receiver for proof results directly from workers. proof_result_rx: CrossbeamReceiver, /// Receives updates from execution and prewarming. @@ -136,13 +137,14 @@ where cancel_rx: CrossbeamReceiver<()>, final_hashed_state_tx: std::sync::mpsc::Sender>, proof_worker_handle: ProofWorkerHandle, + proof_result_tx: ProofResultSender, + proof_result_rx: CrossbeamReceiver, metrics: SparseTrieTaskMetrics, trie: SparseStateTrie, parent_state_root: B256, new_epoch: TrieNodeEpoch, chunk_size: usize, ) -> Self { - let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); let (hashed_state_tx, hashed_state_rx) = crossbeam_channel::unbounded(); let parent_span = tracing::Span::current(); @@ -557,12 +559,13 @@ where &mut self, message: ProofResultMessage, ) -> Result { + let result = message.result?; debug_assert!( self.in_flight_proof_batches > 0, "received proof result without an in-flight proof batch" ); self.in_flight_proof_batches = self.in_flight_proof_batches.saturating_sub(1); - message.result + Ok(result) } fn process_new_updates(&mut self) -> SparseTrieResult<()> { @@ -1117,14 +1120,20 @@ enum SparseTrieTaskMessage { mod tests { use super::*; use alloy_primitives::{keccak256, Address, B256, U256}; + use reth_db_common::init::init_genesis; use reth_provider::{ providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory, - ChainSpecProvider, }; use reth_storage_overlay::OverlayManager; use reth_trie_parallel::proof_task::ProofTaskCtx; use reth_trie_sparse::ArenaParallelSparseTrie; + fn drain_sparse_trie_tasks(runtime: &Runtime) { + for task_name in ["trie-hashing", "storage-workers", "account-workers"] { + runtime.spawn_blocking_named(task_name, || {}).get(); + } + } + #[test] fn test_run_hashing_task_hashed_state_update_forwards() { let (updates_tx, updates_rx) = crossbeam_channel::unbounded(); @@ -1222,14 +1231,19 @@ mod tests { fn run_returns_parent_root_without_revealing_blind_trie_when_no_state_updates() { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); - let anchor_hash = provider_factory.chain_spec().genesis_hash(); + let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); let overlay_factory = OverlayStateProviderFactory::new( provider_factory, OverlayManager::::default() .overlay_builder(anchor_hash), ); - let proof_worker_handle = - ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); + let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); + let proof_worker_handle = ProofWorkerHandle::new( + &runtime, + ProofTaskCtx::new(overlay_factory), + false, + proof_result_tx.clone(), + ); let default_trie = RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); let trie = SparseStateTrie::default() @@ -1246,6 +1260,8 @@ mod tests { cancel_rx, std::sync::mpsc::channel().0, proof_worker_handle, + proof_result_tx, + proof_result_rx, SparseTrieTaskMetrics::default(), trie, parent_state_root, @@ -1261,20 +1277,28 @@ mod tests { assert_eq!(outcome.state_root, parent_state_root); assert!(outcome.trie_updates.is_empty()); assert!(task.trie.state_trie_ref().is_none(), "blind trie should not be revealed"); + + drop(task); + drain_sparse_trie_tasks(&runtime); } #[test] fn stall_check_waits_for_in_flight_proofs_then_reports_pending_updates() { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); - let anchor_hash = provider_factory.chain_spec().genesis_hash(); + let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); let overlay_factory = OverlayStateProviderFactory::new( provider_factory, OverlayManager::::default() .overlay_builder(anchor_hash), ); - let proof_worker_handle = - ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); + let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); + let proof_worker_handle = ProofWorkerHandle::new( + &runtime, + ProofTaskCtx::new(overlay_factory), + false, + proof_result_tx.clone(), + ); let default_trie = RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); let trie = SparseStateTrie::default() @@ -1290,6 +1314,8 @@ mod tests { cancel_rx, std::sync::mpsc::channel().0, proof_worker_handle, + proof_result_tx, + proof_result_rx, SparseTrieTaskMetrics::default(), trie, B256::from([0x55; 32]), @@ -1339,20 +1365,28 @@ mod tests { assert!(!error.contains("pending_storage_leaves")); assert!(!error.contains("pending_account_updates")); assert!(!error.contains(&format!("{slot:?}"))); + + drop(task); + drain_sparse_trie_tasks(&runtime); } #[test] fn run_errors_when_cancel_guard_drops_before_updates_finish() { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); - let anchor_hash = provider_factory.chain_spec().genesis_hash(); + let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); let overlay_factory = OverlayStateProviderFactory::new( provider_factory, OverlayManager::::default() .overlay_builder(anchor_hash), ); - let proof_worker_handle = - ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); + let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); + let proof_worker_handle = ProofWorkerHandle::new( + &runtime, + ProofTaskCtx::new(overlay_factory), + false, + proof_result_tx.clone(), + ); let default_trie = RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); let trie = SparseStateTrie::default() @@ -1368,6 +1402,8 @@ mod tests { cancel_rx, std::sync::mpsc::channel().0, proof_worker_handle, + proof_result_tx, + proof_result_rx, SparseTrieTaskMetrics::default(), trie, B256::from([0x55; 32]), @@ -1383,20 +1419,27 @@ mod tests { assert!(matches!(error, StateRootTaskError::Canceled)); drop(updates_tx); + drop(task); + drain_sparse_trie_tasks(&runtime); } #[test] fn run_ignores_hints_queued_after_updates_finish() { let runtime = reth_tasks::Runtime::test(); let provider_factory = create_test_provider_factory(); - let anchor_hash = provider_factory.chain_spec().genesis_hash(); + let anchor_hash = init_genesis(&provider_factory).expect("failed to initialize genesis"); let overlay_factory = OverlayStateProviderFactory::new( provider_factory, OverlayManager::::default() .overlay_builder(anchor_hash), ); - let proof_worker_handle = - ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false); + let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded(); + let proof_worker_handle = ProofWorkerHandle::new( + &runtime, + ProofTaskCtx::new(overlay_factory), + false, + proof_result_tx.clone(), + ); let default_trie = RevealableSparseTrie::blind_from(ArenaParallelSparseTrie::default()); let trie = SparseStateTrie::default() @@ -1412,6 +1455,8 @@ mod tests { cancel_rx, std::sync::mpsc::channel().0, proof_worker_handle, + proof_result_tx, + proof_result_rx, SparseTrieTaskMetrics::default(), trie, B256::from([0x55; 32]), @@ -1441,5 +1486,8 @@ mod tests { handle.join().unwrap(); assert!(result.expect("state root task stalled on a late hint").is_ok()); + + drop(updates_tx); + drain_sparse_trie_tasks(&runtime); } } diff --git a/crates/trie/parallel/src/error.rs b/crates/trie/parallel/src/error.rs index 2eb349f7700..75ecaa590ee 100644 --- a/crates/trie/parallel/src/error.rs +++ b/crates/trie/parallel/src/error.rs @@ -11,6 +11,9 @@ pub enum StateRootTaskError { /// Proof dispatch error. #[error("proof dispatch failed: {_0}")] ProofDispatch(ProviderError), + /// A proof worker failed before it could process queued work. + #[error("proof worker failed: {_0}")] + ProofWorker(String), /// Sparse trie error. #[error(transparent)] SparseTrie(#[from] SparseTrieError), diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index 6d508e7e4c2..3f98c01dcf1 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -170,6 +170,7 @@ impl ProofWorkerHandle { runtime: &Runtime, task_ctx: ProofTaskCtx, halve_workers: bool, + proof_result_tx: ProofResultSender, ) -> Self where Factory: DatabaseProviderROFactory @@ -180,7 +181,6 @@ impl ProofWorkerHandle { { let (storage_work_tx, storage_work_rx) = unbounded::(); let (account_work_tx, account_work_rx) = unbounded::(); - let cached_storage_roots = Arc::>::default(); let divisor = if halve_workers { 2 } else { 1 }; @@ -206,6 +206,7 @@ impl ProofWorkerHandle { let storage_task_ctx = task_ctx.clone(); let storage_avail = storage_availability.clone(); let storage_roots = cached_storage_roots.clone(); + let storage_result_tx = proof_result_tx.clone(); let storage_parent_span = tracing::Span::current(); runtime.spawn_blocking_named("storage-workers", move || { let worker_id = AtomicUsize::new(0); @@ -237,6 +238,13 @@ impl ProofWorkerHandle { ?error, "Storage worker failed" ); + let _ = storage_result_tx.send(ProofResultMessage { + result: Err(StateRootTaskError::ProofWorker(format!( + "storage worker {worker_id}: {error}" + ))), + elapsed: Duration::ZERO, + state: Default::default(), + }); } }); }); @@ -244,6 +252,7 @@ impl ProofWorkerHandle { let account_rt = runtime.clone(); let account_tx = storage_work_tx.clone(); let account_avail = account_availability.clone(); + let account_result_tx = proof_result_tx; let account_parent_span = tracing::Span::current(); runtime.spawn_blocking_named("account-workers", move || { let worker_id = AtomicUsize::new(0); @@ -276,6 +285,13 @@ impl ProofWorkerHandle { ?error, "Account worker failed" ); + let _ = account_result_tx.send(ProofResultMessage { + result: Err(StateRootTaskError::ProofWorker(format!( + "account worker {worker_id}: {error}" + ))), + elapsed: Duration::ZERO, + state: Default::default(), + }); } }); }); @@ -475,10 +491,11 @@ where } } -/// Channel used by worker threads to deliver `ProofResultMessage` items back to +/// Channel used by worker threads to deliver proof results back to /// `SparseTrieCacheTask`. /// -/// Workers use this sender to deliver proof results directly to `SparseTrieCacheTask`. +/// Workers use this sender to deliver proof results or terminal initialization errors directly to +/// `SparseTrieCacheTask`. pub type ProofResultSender = CrossbeamSender; /// Message containing a completed proof result with metadata for direct delivery to @@ -1183,7 +1200,8 @@ mod tests { let ctx = test_ctx(factory); let runtime = reth_tasks::Runtime::test(); - let proof_handle = ProofWorkerHandle::new(&runtime, ctx, false); + let (proof_result_tx, _) = unbounded(); + let proof_handle = ProofWorkerHandle::new(&runtime, ctx, false, proof_result_tx); // Verify handle can be cloned let _cloned_handle = proof_handle.clone(); From 0bba0c6880c39aed47c9d689962ede7f944be7c5 Mon Sep 17 00:00:00 2001 From: Aliaksei Osipau Date: Fri, 31 Jul 2026 18:14:22 +0300 Subject: [PATCH 076/148] fix(net): recover partial header responses (#26482) Co-authored-by: Matthias Seitz --- .../src/headers/reverse_headers.rs | 200 +++++++++++++++++- crates/net/downloaders/src/metrics.rs | 2 + crates/net/network/src/fetch/mod.rs | 35 +++ crates/net/p2p/src/error.rs | 18 +- 4 files changed, 246 insertions(+), 9 deletions(-) diff --git a/crates/net/downloaders/src/headers/reverse_headers.rs b/crates/net/downloaders/src/headers/reverse_headers.rs index 6a866f0ad9b..4d1ea614c26 100644 --- a/crates/net/downloaders/src/headers/reverse_headers.rs +++ b/crates/net/downloaders/src/headers/reverse_headers.rs @@ -475,17 +475,19 @@ where .into()) } - if (headers.len() as u64) != request.limit { + let received_headers = headers.len() as u64; + if received_headers > request.limit { return Err(HeadersResponseError { peer_id: Some(peer_id), - error: DownloadError::HeadersResponseTooShort(GotExpected { - got: headers.len() as u64, + error: DownloadError::HeadersResponseTooLong(GotExpected { + got: received_headers, expected: request.limit, }), request, } .into()) } + let missing_headers = request.limit - received_headers; // sort headers from highest to lowest block number headers.sort_unstable_by_key(|h| Reverse(h.number())); @@ -511,6 +513,14 @@ where if highest.number() == self.next_chain_tip_block_number { // is next response, validate it self.process_next_headers(request, headers, peer_id)?; + // request the missing headers before validating buffered responses, because a + // buffered validation error returns early and would otherwise leave the + // remainder of this range unrequested + self.requeue_missing_headers( + requested_block_number, + received_headers, + missing_headers, + ); // try to validate all buffered responses blocked by this successful response self.try_validate_buffered() .map(Err::<(), ReverseHeadersDownloaderError>) @@ -522,7 +532,12 @@ where headers, request, peer_id, - }) + }); + self.requeue_missing_headers( + requested_block_number, + received_headers, + missing_headers, + ); } Ok(()) @@ -598,6 +613,29 @@ where self.metrics.in_flight_requests.increment(1.); } + /// Submits a high-priority request for the missing suffix of a partial response. + /// + /// Peers are allowed to respond with fewer headers than requested, for example due to + /// response size limits. Expects that the caller has verified that the response started at + /// `requested_block_number` and contained `received_headers` headers. + fn requeue_missing_headers( + &self, + requested_block_number: u64, + received_headers: u64, + missing_headers: u64, + ) { + if missing_headers > 0 { + self.metrics.partial_responses.increment(1); + self.submit_request( + HeadersRequest::falling( + (requested_block_number - received_headers).into(), + missing_headers, + ), + Priority::High, + ); + } + } + fn request_fut( &self, request: HeadersRequest, @@ -1251,7 +1289,74 @@ mod tests { use alloy_eips::{eip1898::BlockWithParent, BlockNumHash}; use assert_matches::assert_matches; use reth_consensus::test_utils::TestConsensus; - use reth_network_p2p::test_utils::TestHeadersClient; + use reth_network_p2p::{ + download::DownloadClient, error::PeerRequestResult, test_utils::TestHeadersClient, + }; + use reth_network_peers::WithPeerId; + use std::sync::{ + atomic::{AtomicU64, Ordering as AtomicOrdering}, + Mutex, + }; + + #[derive(Clone, Debug)] + struct CappedHeadersClient { + responses: Arc>>, + requests: Arc>>, + bad_messages: Arc, + response_limit: usize, + } + + impl CappedHeadersClient { + fn new(responses: Vec

, response_limit: usize) -> Self { + Self { + responses: Arc::new(Mutex::new(responses)), + requests: Default::default(), + bad_messages: Default::default(), + response_limit, + } + } + + fn numeric_requests(&self) -> Vec<(u64, u64)> { + self.requests + .lock() + .unwrap() + .iter() + .filter_map(|request| request.start.as_number().map(|start| (start, request.limit))) + .collect() + } + + fn bad_message_count(&self) -> u64 { + self.bad_messages.load(AtomicOrdering::SeqCst) + } + } + + impl DownloadClient for CappedHeadersClient { + fn report_bad_message(&self, _peer_id: PeerId) { + self.bad_messages.fetch_add(1, AtomicOrdering::SeqCst); + } + + fn num_connected_peers(&self) -> usize { + 1 + } + } + + impl HeadersClient for CappedHeadersClient { + type Header = Header; + type Output = futures::future::Ready>>; + + fn get_headers_with_priority( + &self, + request: HeadersRequest, + _priority: Priority, + ) -> Self::Output { + self.requests.lock().unwrap().push(request.clone()); + let mut responses = self.responses.lock().unwrap(); + let response_length = request.limit.min(self.response_limit as u64) as usize; + let response = responses.drain(..response_length).collect(); + + futures::future::ready(Ok(WithPeerId::new(PeerId::default(), response))) + } + } /// Tests that `replace_number` works the same way as `Option::replace` #[test] @@ -1381,6 +1486,14 @@ mod tests { assert_eq!(request.limit, 1); } + #[test] + fn default_request_limit_matches_sync_config() { + assert_eq!( + ReverseHeadersDownloaderBuilder::default().request_limit, + HeadersConfig::default().downloader_request_limit + ); + } + /// Tests that request calc works #[test] fn test_next_request() { @@ -1547,4 +1660,81 @@ mod tests { assert!(downloader.next().await.is_none()); } + + #[tokio::test] + async fn downloads_capped_header_responses() { + let p3 = SealedHeader::default(); + let p2 = child_header(&p3); + let p1 = child_header(&p2); + let p0 = child_header(&p1); + + let client = CappedHeadersClient::new( + vec![p0.as_ref().clone(), p1.as_ref().clone(), p2.as_ref().clone()], + 1, + ); + let mut downloader = ReverseHeadersDownloaderBuilder::default() + .stream_batch_size(1) + .request_limit(2) + .min_concurrent_requests(1) + .max_concurrent_requests(1) + .build(client.clone(), Arc::new(TestConsensus::default())); + downloader.update_local_head(p3); + downloader.update_sync_target(SyncTarget::Tip(p0.hash())); + + assert_eq!(downloader.next().await.unwrap().unwrap(), vec![p0]); + assert_eq!(downloader.next().await.unwrap().unwrap(), vec![p1]); + assert_eq!(downloader.next().await.unwrap().unwrap(), vec![p2]); + assert!(downloader.next().await.is_none()); + + assert_eq!(client.numeric_requests(), vec![(2, 2), (1, 1)]); + assert_eq!(client.bad_message_count(), 0); + } + + #[test] + fn requeues_missing_headers_for_buffered_partial_response() { + let client = CappedHeadersClient::new(Vec::new(), 0); + let mut downloader = ReverseHeadersDownloaderBuilder::default() + .build(client.clone(), Arc::new(TestConsensus::default())); + downloader.local_head = Some(SealedHeader::default()); + downloader.sync_target = Some(SyncTargetBlock::from_number(10)); + downloader.next_chain_tip_block_number = 10; + + // a partial response that can't be validated yet is buffered, but the missing suffix must + // be requested immediately + let request = HeadersRequest::falling(5u64.into(), 3); + let response = vec![Header { number: 5, ..Default::default() }]; + let outcome = downloader.on_headers_outcome(HeadersRequestOutcome { + request, + outcome: Ok(WithPeerId::new(PeerId::default(), response)), + }); + + assert!(outcome.is_ok()); + assert_eq!(downloader.buffered_responses.len(), 1); + assert_eq!(client.numeric_requests(), vec![(4, 2)]); + assert_eq!(client.bad_message_count(), 0); + } + + #[test] + fn rejects_over_long_headers_response() { + let client = CappedHeadersClient::new(Vec::new(), 0); + let mut downloader = ReverseHeadersDownloaderBuilder::default() + .build(client.clone(), Arc::new(TestConsensus::default())); + + let request = HeadersRequest::falling(5u64.into(), 1); + let response = vec![ + Header { number: 5, ..Default::default() }, + Header { number: 4, ..Default::default() }, + ]; + let outcome = downloader.on_headers_outcome(HeadersRequestOutcome { + request, + outcome: Ok(WithPeerId::new(PeerId::default(), response)), + }); + + assert_matches!( + outcome, + Err(ReverseHeadersDownloaderError::Response(err)) + if matches!(err.error, DownloadError::HeadersResponseTooLong(_)) + ); + assert!(client.numeric_requests().is_empty()); + } } diff --git a/crates/net/downloaders/src/metrics.rs b/crates/net/downloaders/src/metrics.rs index 0d0ba36079c..56990b6d7e9 100644 --- a/crates/net/downloaders/src/metrics.rs +++ b/crates/net/downloaders/src/metrics.rs @@ -103,6 +103,8 @@ pub struct HeaderDownloaderMetrics { pub total_flushed: Counter, /// Number of items that were successfully downloaded pub total_downloaded: Counter, + /// Number of responses that contained fewer headers than requested + pub partial_responses: Counter, /// The number of requests (can contain more than 1 item) currently in-flight. pub in_flight_requests: Gauge, /// The number of responses (can contain more than 1 item) in the internal buffer of the diff --git a/crates/net/network/src/fetch/mod.rs b/crates/net/network/src/fetch/mod.rs index 8dd7bdd991c..d6074df6420 100644 --- a/crates/net/network/src/fetch/mod.rs +++ b/crates/net/network/src/fetch/mod.rs @@ -1057,6 +1057,41 @@ mod tests { assert!(fetcher.peers[&peer_id].state.is_idle()); } + #[tokio::test] + async fn test_partial_header_response_triggers_followup() { + let (mut fetcher, peer_id) = fetcher_with_peer(); + + let (followup_tx, _followup_rx) = oneshot::channel(); + fetcher.queued_requests.push_back(DownloadRequest::GetBlockHeaders { + request: HeadersRequest::falling(1u64.into(), 1), + response: followup_tx, + priority: Priority::High, + }); + + let (response_tx, mut response_rx) = oneshot::channel(); + fetcher.inflight_headers_requests.insert( + peer_id, + Request { request: HeadersRequest::falling(2u64.into(), 2), response: response_tx }, + ); + fetcher.peers.get_mut(&peer_id).unwrap().state = PeerState::GetBlockHeaders; + + let header = Header { number: 2, ..Default::default() }; + let outcome = fetcher.on_block_headers_response(peer_id, Ok(vec![header])); + + let Some(BlockResponseOutcome::Request( + dispatched_peer, + BlockRequest::GetBlockHeaders(request), + )) = outcome + else { + panic!("expected a header followup request") + }; + assert_eq!(dispatched_peer, peer_id); + assert_eq!(request.start_block, 1u64.into()); + assert_eq!(request.limit, 1); + assert!(!fetcher.peers[&peer_id].last_response_likely_bad); + assert!(response_rx.try_recv().is_ok()); + } + #[test] fn test_peer_is_better_none_requirement() { let peer1 = Peer { diff --git a/crates/net/p2p/src/error.rs b/crates/net/p2p/src/error.rs index 609121cad5e..36708c7c2e6 100644 --- a/crates/net/p2p/src/error.rs +++ b/crates/net/p2p/src/error.rs @@ -33,7 +33,7 @@ impl EthResponseValidator for RequestResult> { Ok(headers) => { let request_length = headers.len() as u64; - if request_length <= 1 && request.limit != request_length { + if (request_length == 0 && request.limit > 0) || request_length > request.limit { return true } @@ -154,9 +154,9 @@ pub enum DownloadError { /// Received a response to a request with unexpected start block #[display("headers response starts at unexpected block: {_0}")] HeadersResponseStartBlockMismatch(GotExpected), - /// Received headers with less than expected items. - #[display("received less headers than expected: {_0}")] - HeadersResponseTooShort(GotExpected), + /// Received more headers than requested. + #[display("received more headers than requested: {_0}")] + HeadersResponseTooLong(GotExpected), /* ==================== BODIES ERRORS ==================== */ /// Block validation failed @@ -232,5 +232,15 @@ mod tests { HeadersRequest { start: 0u64.into(), limit: 1, direction: Default::default() }; let headers: Vec
= vec![]; assert!(Ok(headers).is_likely_bad_headers_response(&request)); + + let request = + HeadersRequest { start: 0u64.into(), limit: 2, direction: Default::default() }; + let headers = vec![Header::default()]; + assert!(!Ok(headers).is_likely_bad_headers_response(&request)); + + let request = + HeadersRequest { start: 0u64.into(), limit: 1, direction: Default::default() }; + let headers = vec![Header::default(), Header::default()]; + assert!(Ok(headers).is_likely_bad_headers_response(&request)); } } From 8bfea20a911f3724000c5e927a3763ef84cf8cac Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 31 Jul 2026 20:45:24 +0200 Subject: [PATCH 077/148] fix(rpc): stop cancelled payload body range requests (#26546) --- crates/rpc/rpc-engine-api/src/engine_api.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 27aa6916542..126ccb25c1d 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -650,6 +650,10 @@ where // Check if the requested range starts before the earliest available block due to pruning/expiry let earliest_block = inner.provider.earliest_block_number().unwrap_or(0); for num in start..=end { + if tx.is_closed() { + return; + } + if num < earliest_block { result.push(None); continue; From 0450ea3d867b84e41a3b230b5839a640f567af64 Mon Sep 17 00:00:00 2001 From: Roman Krasiuk Date: Mon, 3 Aug 2026 11:13:16 +0200 Subject: [PATCH 078/148] feat(rpc): add eth_getMultiProof (#26555) --- crates/rpc/rpc-api/src/engine.rs | 8 +++ crates/rpc/rpc-builder/tests/it/http.rs | 12 +++++ crates/rpc/rpc-eth-api/src/core.rs | 18 +++++++ crates/rpc/rpc-eth-api/src/helpers/state.rs | 56 ++++++++++++++++++++- crates/rpc/rpc/src/engine.rs | 9 ++++ 5 files changed, 102 insertions(+), 1 deletion(-) diff --git a/crates/rpc/rpc-api/src/engine.rs b/crates/rpc/rpc-api/src/engine.rs index d50dc068fd2..e3714d6ecf1 100644 --- a/crates/rpc/rpc-api/src/engine.rs +++ b/crates/rpc/rpc-api/src/engine.rs @@ -422,6 +422,14 @@ pub trait EngineEthApi { block_number: Option, ) -> RpcResult; + /// Returns the account and storage values of the specified targets including Merkle proofs. + #[method(name = "getMultiProof")] + async fn get_multi_proof( + &self, + targets: Vec<(Address, Vec)>, + block_number: Option, + ) -> RpcResult>; + /// Returns the EIP-7928 block access list for a block by hash. #[method(name = "getBlockAccessListByBlockHash")] async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult>; diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 4b341c9c86b..7bec44f4fcc 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -381,6 +381,18 @@ where ) .await .unwrap(); + let proofs = EthApiClient::< + TransactionRequest, + Transaction, + Block, + Receipt, + Header, + TransactionSigned, + >::get_multi_proof(client, vec![(address, vec![B256::ZERO])], None) + .await + .unwrap(); + assert_eq!(proofs.len(), 1); + assert_eq!(proofs[0].address, address); // Unimplemented assert!( diff --git a/crates/rpc/rpc-eth-api/src/core.rs b/crates/rpc/rpc-eth-api/src/core.rs index 7f55f338339..78a6c90fcb7 100644 --- a/crates/rpc/rpc-eth-api/src/core.rs +++ b/crates/rpc/rpc-eth-api/src/core.rs @@ -415,6 +415,14 @@ pub trait EthApi< block_number: Option, ) -> RpcResult; + /// Returns the account and storage values of the specified targets including Merkle proofs. + #[method(name = "getMultiProof")] + async fn get_multi_proof( + &self, + targets: Vec<(Address, Vec)>, + block_number: Option, + ) -> RpcResult>; + /// Returns the account's balance, nonce, and code. /// /// This is similar to `eth_getAccount` but does not return the storage root. @@ -954,6 +962,16 @@ where Ok(EthState::get_proof(self, address, keys, block_number)?.await?) } + /// Handler for: `eth_getMultiProof` + async fn get_multi_proof( + &self, + targets: Vec<(Address, Vec)>, + block_number: Option, + ) -> RpcResult> { + trace!(target: "rpc::eth", ?targets, ?block_number, "Serving eth_getMultiProof"); + Ok(EthState::get_multi_proof(self, targets, block_number)?.await?) + } + /// Handler for: `eth_getAccountInfo` async fn get_account_info( &self, diff --git a/crates/rpc/rpc-eth-api/src/helpers/state.rs b/crates/rpc/rpc-eth-api/src/helpers/state.rs index 8c9a109811e..4dc41576c98 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/state.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/state.rs @@ -5,7 +5,7 @@ use super::{EthApiSpec, LoadBlock, LoadPendingBlock, SpawnBlocking}; use crate::{EthApiTypes, FromEthApiError, RpcNodeCore, RpcNodeCoreExt}; use alloy_consensus::constants::KECCAK_EMPTY; use alloy_eips::BlockId; -use alloy_primitives::{Address, Bytes, B256, U256}; +use alloy_primitives::{keccak256, Address, Bytes, B256, U256}; use alloy_rpc_types_eth::{Account, AccountInfo, EIP1186AccountProofResponse}; use alloy_serde::JsonStorageKey; use futures::Future; @@ -22,6 +22,7 @@ use reth_storage_api::{ BlockIdReader, BlockReaderIdExt, StateProvider, StateProviderBox, StateProviderFactory, }; use reth_transaction_pool::TransactionPool; +use reth_trie_common::MultiProofTargets; use std::{collections::HashMap, sync::Arc}; /// Helper methods for `eth_` methods relating to state (accounts). @@ -186,6 +187,59 @@ pub trait EthState: LoadState + SpawnBlocking { }) } + /// Returns account and storage proofs for multiple targets at the given block number. + fn get_multi_proof( + &self, + targets: Vec<(Address, Vec)>, + block_id: Option, + ) -> Result< + impl Future, Self::Error>> + Send, + Self::Error, + > + where + Self: EthApiSpec, + { + Ok(async move { + let _permit = self + .acquire_owned_tracing() + .await + .map_err(RethError::other) + .map_err(EthApiError::Internal)?; + + let block_id = block_id.unwrap_or_default(); + self.ensure_within_proof_window(block_id)?; + + self.spawn_blocking_io_fut(async move |this| { + let state = this.state_at_block_id(block_id).await?; + let mut proof_targets = MultiProofTargets::with_capacity(targets.len()); + for (address, slots) in &targets { + proof_targets + .entry(keccak256(address)) + .or_default() + .extend(slots.iter().map(keccak256)); + } + + let multiproof = state + .multiproof(Default::default(), proof_targets) + .map_err(Self::Error::from_eth_err)?; + + targets + .into_iter() + .map(|(address, slots)| { + let proof = multiproof + .account_proof(address, &slots) + .map_err(RethError::other) + .map_err(Self::Error::from_eth_err)?; + let storage_keys = + slots.into_iter().map(JsonStorageKey::from).collect::>(); + Ok(proof.into_eip1186_response(storage_keys)) + }) + .collect::, Self::Error>>() + }) + .await + }) + } + /// Returns the account at the given address for the provided block identifier. fn get_account( &self, diff --git a/crates/rpc/rpc/src/engine.rs b/crates/rpc/rpc/src/engine.rs index e1d64911a8b..d8409c2694b 100644 --- a/crates/rpc/rpc/src/engine.rs +++ b/crates/rpc/rpc/src/engine.rs @@ -147,6 +147,15 @@ where self.eth.get_proof(address, keys, block_number).instrument(engine_span!()).await } + /// Handler for `eth_getMultiProof` + async fn get_multi_proof( + &self, + targets: Vec<(Address, Vec)>, + block_number: Option, + ) -> Result> { + self.eth.get_multi_proof(targets, block_number).instrument(engine_span!()).await + } + /// Handler for `eth_getBlockAccessListByBlockHash` async fn block_access_list_by_block_hash(&self, hash: B256) -> Result> { self.eth.block_access_list_by_block_hash(hash).instrument(engine_span!()).await From a5c244e7020773b9549fd31fb3d9cb6177bbdc49 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Aug 2026 11:55:48 +0200 Subject: [PATCH 079/148] chore(deps): bump the ci-weekly group with 3 updates (#26556) Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/bench-scheduled.yml | 2 +- .github/workflows/bench.yml | 2 +- .github/workflows/e2e.yml | 4 ++-- .github/workflows/integration.yml | 4 ++-- .github/workflows/lint.yml | 8 ++++---- .github/workflows/stale.yml | 2 +- .github/workflows/unit.yml | 4 ++-- 7 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/bench-scheduled.yml b/.github/workflows/bench-scheduled.yml index 5f8d955bdd1..9d9fa094b41 100644 --- a/.github/workflows/bench-scheduled.yml +++ b/.github/workflows/bench-scheduled.yml @@ -351,7 +351,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 continue-on-error: true - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - name: Install dependencies env: diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index f88d69b989a..956652f17e9 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -931,7 +931,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 continue-on-error: true - - uses: astral-sh/setup-uv@11f9893b081a58869d3b5fccaea48c9e9e46f990 # v8.3.2 + - uses: astral-sh/setup-uv@c771a70e6277c0a99b617c7a806ffedaca235ff9 # v9.0.0 - name: Mask OTLP endpoints uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index cd7f3450078..1abce78da42 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -36,7 +36,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 @@ -71,7 +71,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index a0afb73c59d..591bc31304e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -44,7 +44,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable - name: Install Geth run: .github/scripts/install_geth.sh - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -85,7 +85,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 6081c0c2439..562185f766b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -83,7 +83,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -109,7 +109,7 @@ jobs: - uses: dtolnay/rust-toolchain@stable with: target: riscv32imac-unknown-none-elf - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -137,7 +137,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-hack - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 @@ -227,7 +227,7 @@ jobs: - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: cache-on-failure: true - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-udeps - run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index 066b753ff59..fa32d4f4089 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -17,7 +17,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: days-before-stale: 21 days-before-close: 7 diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 41274c56cb3..74986cbb3b9 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -47,7 +47,7 @@ jobs: with: cache-targets: false cache-on-failure: true - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest - if: "${{ matrix.type == 'book' }}" @@ -97,7 +97,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable - - uses: taiki-e/install-action@a6b2e2dcd845ddd7f509ce4f3ed3d922b80cc5d9 # v2.84.0 + - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 From 7a98a9c7ac857b14fb07a2014fe1fd5ca2868523 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 3 Aug 2026 18:03:33 +0200 Subject: [PATCH 080/148] chore(deps): update Alloy and Tokio Tungstenite (#26562) --- Cargo.lock | 161 +++++++++++++++++++++----------------------- Cargo.toml | 57 ++++++++-------- bin/reth/Cargo.toml | 2 +- 3 files changed, 107 insertions(+), 113 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 8a217855dbe..d1b7815be38 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -121,9 +121,9 @@ dependencies = [ [[package]] name = "alloy-consensus" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f0f42ee16e9eaffa4df5aa3a17b1df88dd222310d13152ab91357a2551fd054" +checksum = "51b9eb70841f76b69b1205bb57b551b7d7afd8407ad57cbda02e833877507c16" dependencies = [ "alloy-eips", "alloy-primitives", @@ -149,9 +149,9 @@ dependencies = [ [[package]] name = "alloy-consensus-any" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db388e804269d977d60fa7530ccfefe9184ca48d4055dad3ec7e649a29186a06" +checksum = "f89486e1bd2c5c56522cf18e73460c3fe7b54d50f5e80131466e62b7632edfcb" dependencies = [ "alloy-consensus", "alloy-eips", @@ -164,9 +164,9 @@ dependencies = [ [[package]] name = "alloy-contract" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69937abc65310c6eb739e540b107688943b61f38e7dcf9fde769367d3608d657" +checksum = "b4c0dadb2468adc8aaaaad4c9a6cfa6bd055fe6596c40017067b37671f2c45f5" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -265,9 +265,9 @@ dependencies = [ [[package]] name = "alloy-eips" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52010e7c255f45b0ef3ca18144cbfbe0c044b055fe53f59323f3df5d95f265b1" +checksum = "fddc9c1868d6331b56926de8ab4c4a80d8dcb7bc4f8f7bebc6a89d0548f79af2" dependencies = [ "alloy-eip2124", "alloy-eip2930", @@ -311,9 +311,9 @@ dependencies = [ [[package]] name = "alloy-genesis" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a60fa6337a6af8bae818de3030448827bf55c1321c90c7d1f6f3b79257ead2a" +checksum = "921b3dccecc4e9dd571c9f9a728d913c2fc7a8ecdfce6005ed99751817523d0d" dependencies = [ "alloy-eips", "alloy-primitives", @@ -365,9 +365,9 @@ dependencies = [ [[package]] name = "alloy-json-rpc" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d7ff4d7f1503d12edda9b241904a309d6e1d5bcee3f0ffdc4d3d3f16f366b8" +checksum = "989d701ece9bccec3294991ce4b4a39dd1cecd0979ea58e86b9878679591b4f5" dependencies = [ "alloy-primitives", "alloy-sol-types", @@ -380,9 +380,9 @@ dependencies = [ [[package]] name = "alloy-network" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "44ff1439d901b44f6ff840545f82d65e66830ee33fb33cae495a566d66e0ccc8" +checksum = "acfca49fbbe3b30de8114b9cfd89029f87cb33ca7e8a5f5f48a809caaf085126" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -406,9 +406,9 @@ dependencies = [ [[package]] name = "alloy-network-primitives" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb920286c4a71fb0ff3bbb207a640a531a42fceef5b43d4a943f7d5265789876" +checksum = "2af7b76520abc91b9e0f1aa987d42df55a66494f94a1881cdadd9b8f67e0b924" dependencies = [ "alloy-consensus", "alloy-eips", @@ -419,9 +419,9 @@ dependencies = [ [[package]] name = "alloy-node-bindings" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd2d06c4044c3ae7f9b7a70c9a7bb3bde9a08e99f4d9aa91001e17a182b562dc" +checksum = "48b81973f768c49fe233d6e43da093765d60c1f9bcc81c82ba4ef31c8e273b4b" dependencies = [ "alloy-genesis", "alloy-hardforks 0.2.13", @@ -473,9 +473,9 @@ dependencies = [ [[package]] name = "alloy-provider" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "329375f50202cc6b2dcc8c6b1e53b716cbbebbeea54e931befc4f3b628dfc469" +checksum = "3a3fb53b31526b0f97cdbfaf9d8bb72b3e4a9ac7965148d79ecec4c733c8558e" dependencies = [ "alloy-chains", "alloy-consensus", @@ -517,9 +517,9 @@ dependencies = [ [[package]] name = "alloy-pubsub" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71044e076a7c243454619e36279fc4654bc1ff846fbf666663ef574388859c84" +checksum = "197325ef17fcdf5dfa21999a2a33877b82bb5009d5cc9b5c7f5ddb840fa3b6bb" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -561,9 +561,9 @@ dependencies = [ [[package]] name = "alloy-rpc-client" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52bffef2c5714f578f267bb959b47908b95728f1f3e2073a96d928f4441c148d" +checksum = "f5274f9ada7f558d2feaba16fadcb96022512566cf424147f9cfa2000bc2e458" dependencies = [ "alloy-json-rpc", "alloy-primitives", @@ -587,9 +587,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a64449cae1cf37c4907b10ccca1613b71809f9a1f3ad6e2b4cc14d08e2e4947f" +checksum = "12a456d697d4f639d20ac242981de13b94f911c99199e4175e85eaf52b8f785c" dependencies = [ "alloy-primitives", "alloy-rpc-types-engine", @@ -600,9 +600,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-admin" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "980671fae7bff5ab0ae2d0bfe5c97e42cda103fece106f8d0e584a41c789fc7c" +checksum = "7f36ee47a0f4e901b95180fbe86c92fec3ea73f12c88c5ec70d1cb41abb8d911" dependencies = [ "alloy-genesis", "alloy-primitives", @@ -612,9 +612,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-anvil" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261411a259d346a69078237e2997a7ce0faba006baf5b1937023316d5ef07bf4" +checksum = "7d7e911c628067c406effef5137d0e253d98e117cdb81d47aacfe2816199297e" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -624,9 +624,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-any" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fe7fa70c3af4b2de7f670b50d680700e0f979ce3a0ab7725958b51ffc3b16c2" +checksum = "5493c88bfdd02271de47f28d69481191a2af6b2907901a3a8c7aff79a6b574eb" dependencies = [ "alloy-consensus-any", "alloy-network-primitives", @@ -639,9 +639,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-beacon" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d22759d1d0e79082a5a06f75dc45bf55dfedbb4ac9890da1f179d85a5d2f32f7" +checksum = "b8573342cde98a4d40120b7bc0063fbda42cfd87fb305ab834704177b60d8691" dependencies = [ "alloy-eips", "alloy-primitives", @@ -659,9 +659,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-debug" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b918cbaadbdcffa4ed64840a5ecbde80a6476ab23adc8c5c8ae6feec081058c" +checksum = "bddf06191e2b12eaaeb3a50ef29a8f3416fb74c61421b80f9d7554c4c1d59a56" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -672,9 +672,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-engine" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d30ff2e8f7d05433c1d36cf0f1ba045bf071a3d139f63923742fbdfb171dfe44" +checksum = "866fa880cfd95681c979bf3f8385964c2317ef7423c31c0de53159f01112fedf" dependencies = [ "alloy-consensus", "alloy-eips", @@ -693,9 +693,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-eth" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04f60a196df184facc9acbb61bd765a36cfb3c9d27eb6b43d5bd26340df1b96a" +checksum = "9d3b805bb9cafaf1362c28b377e2035e49fc5ae0a5ee3ea61bafd9a5d33959e1" dependencies = [ "alloy-consensus", "alloy-consensus-any", @@ -715,9 +715,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-mev" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b04a8f30a439c995d71ab999d56a828dcbe4d90b4f012419832779e60acf86af" +checksum = "2b973745b97a364c0215f681b2fb3cb6f0bbeda4dcdba5f56c50f999719ec96f" dependencies = [ "alloy-consensus", "alloy-eips", @@ -730,9 +730,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-trace" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a12cafc0ee2290122ec4e069bb0f2389a356f4b5414f2e82de8288049c2bc98a" +checksum = "2a6fae28f1bda2586b117a5754b49366160770e63e09ac93fc3d872015a5a993" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -744,9 +744,9 @@ dependencies = [ [[package]] name = "alloy-rpc-types-txpool" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "157eec8f8661e6e70e3749c0a4d918a8831f0b0fc152aed60c0de457faca420b" +checksum = "4962825b8819e54028cbdaadeba128a613afa55c91b9a88e0b0dc257772a14d7" dependencies = [ "alloy-primitives", "alloy-rpc-types-eth", @@ -756,9 +756,9 @@ dependencies = [ [[package]] name = "alloy-serde" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07758cbde42596b66da52bb661d77e253c3f7f9b3f553faf07adbadeb6419502" +checksum = "3ad7efdd5e8fe92f5bab87956d6d3f717d1614c4ab6ad0d1f869b16e83b0dff9" dependencies = [ "alloy-primitives", "arbitrary", @@ -768,9 +768,9 @@ dependencies = [ [[package]] name = "alloy-signer" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc43a73bc1bf7044898e1b0dcefaba45d0cfac3e379bb5ca29cc3e3a04c61451" +checksum = "afa150cd076471ef21aa7b181d8186cb86c3239518859a2e5d80fbec74f0ec38" dependencies = [ "alloy-primitives", "async-trait", @@ -783,9 +783,9 @@ dependencies = [ [[package]] name = "alloy-signer-local" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "372ff9be2b61c2cd6d33cfcef9bd5865962c0628c932767f5154443aba2c9746" +checksum = "f125bb730d4772538835d4a31e8e5ed5068e2a991619ba32711fd53ad8c93032" dependencies = [ "alloy-consensus", "alloy-network", @@ -872,9 +872,9 @@ dependencies = [ [[package]] name = "alloy-transport" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830c87b05870c80c8d2cfb9245feaff36468893f19fc124c7ff563cabda6386c" +checksum = "3cd5dbd32cae5d1bc3afaf455064ebd9bcce70b57de220468908717c225225dd" dependencies = [ "alloy-json-rpc", "auto_impl", @@ -895,9 +895,9 @@ dependencies = [ [[package]] name = "alloy-transport-http" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23ecd298fb0e6a31bd90c72efba2728e3d7569f1c89a3719576fb0e4e40b0fc8" +checksum = "f0b7b1f0e528ae788d05937f5b351afae3b4c1e08f17a061d868bd28e56aa9e3" dependencies = [ "alloy-json-rpc", "alloy-transport", @@ -911,9 +911,9 @@ dependencies = [ [[package]] name = "alloy-transport-ipc" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7b51470da19d0ed256bb3cf31042cceca3ce01714a9ca76624fc71a999f6a27" +checksum = "b964b4c048eea647a701a6af07307c64995c4abd42d46ba0b3be9593ae3c333c" dependencies = [ "alloy-json-rpc", "alloy-pubsub", @@ -931,9 +931,9 @@ dependencies = [ [[package]] name = "alloy-transport-ws" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67177a75a6e8a44f39ccbbc64a3e3a70186ef401d7313285b56735d4285aa34" +checksum = "efecf9efc5d56276e115b13b36ff27311b8cd6f72affc9788a10ff46b55b2616" dependencies = [ "alloy-pubsub", "alloy-transport", @@ -970,9 +970,9 @@ dependencies = [ [[package]] name = "alloy-tx-macros" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e50185b84799c8b5f016a0c9da74309fe28d1c70fbe590a4e8d9cae6df3eebb0" +checksum = "b3e2386593cef5b12c7d439dbcf62d7d51c3a50dc943909b42ae34b48f1b69dc" dependencies = [ "darling 0.23.0", "proc-macro2", @@ -1031,7 +1031,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1042,7 +1042,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1965,7 +1965,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.23.0", + "darling 0.21.3", "ident_case", "prettyplease", "proc-macro2", @@ -3427,7 +3427,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5171,7 +5171,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6248,7 +6248,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11369,7 +11369,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11449,7 +11449,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12082,7 +12082,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -12330,7 +12330,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -12647,9 +12647,9 @@ dependencies = [ [[package]] name = "tokio-tungstenite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857" +checksum = "8f72a05e828585856dacd553fba484c242c46e391fb0e58917c942ee9202915c" dependencies = [ "futures-util", "log", @@ -13088,9 +13088,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "tungstenite" -version = "0.28.0" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8628dcc84e5a09eb3d8423d6cb682965dea9133204e8fb3efee74c2a0c259442" +checksum = "6c01152af293afb9c7c2a57e4b559c5620b421f6d133261c60dd2d0cdb38e6b8" dependencies = [ "bytes", "data-encoding", @@ -13102,7 +13102,6 @@ dependencies = [ "rustls-pki-types", "sha1", "thiserror 2.0.18", - "utf-8", ] [[package]] @@ -13235,12 +13234,6 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "utf-8" -version = "0.7.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09cc8ee72d2a9becf2f2febe0205bbed8fc6615b7cb429ad062dc7b7ddd036a9" - [[package]] name = "utf16_iter" version = "1.0.5" @@ -13600,7 +13593,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 4815b98a73f..bae4ee21f90 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -447,33 +447,34 @@ alloy-trie = { version = "0.9.4", default-features = false } alloy-hardforks = "0.4.7" -alloy-consensus = { version = "2.2.0", default-features = false } -alloy-contract = { version = "2.2.0", default-features = false } -alloy-eips = { version = "2.2.0", default-features = false } -alloy-genesis = { version = "2.2.0", default-features = false } -alloy-json-rpc = { version = "2.2.0", default-features = false } -alloy-network = { version = "2.2.0", default-features = false } -alloy-network-primitives = { version = "2.2.0", default-features = false } -alloy-provider = { version = "2.2.0", features = ["reqwest", "debug-api"], default-features = false } -alloy-pubsub = { version = "2.2.0", default-features = false } -alloy-rpc-client = { version = "2.2.0", default-features = false } -alloy-rpc-types = { version = "2.2.0", features = ["eth"], default-features = false } -alloy-rpc-types-admin = { version = "2.2.0", default-features = false } -alloy-rpc-types-anvil = { version = "2.2.0", default-features = false } -alloy-rpc-types-beacon = { version = "2.2.0", default-features = false } -alloy-rpc-types-debug = { version = "2.2.0", default-features = false } -alloy-rpc-types-engine = { version = "2.2.0", default-features = false } -alloy-rpc-types-eth = { version = "2.2.0", default-features = false } -alloy-rpc-types-mev = { version = "2.2.0", default-features = false } -alloy-rpc-types-trace = { version = "2.2.0", default-features = false } -alloy-rpc-types-txpool = { version = "2.2.0", default-features = false } -alloy-serde = { version = "2.2.0", default-features = false } -alloy-signer = { version = "2.2.0", default-features = false } -alloy-signer-local = { version = "2.2.0", default-features = false } -alloy-transport = { version = "2.2.0" } -alloy-transport-http = { version = "2.2.0", features = ["reqwest-rustls-tls"], default-features = false } -alloy-transport-ipc = { version = "2.2.0", default-features = false } -alloy-transport-ws = { version = "2.2.0", default-features = false } +alloy-consensus = { version = "2.3.0", default-features = false } +alloy-contract = { version = "2.3.0", default-features = false } +alloy-eips = { version = "2.3.0", default-features = false } +alloy-genesis = { version = "2.3.0", default-features = false } +alloy-json-rpc = { version = "2.3.0", default-features = false } +alloy-network = { version = "2.3.0", default-features = false } +alloy-network-primitives = { version = "2.3.0", default-features = false } +alloy-node-bindings = "2.3.0" +alloy-provider = { version = "2.3.0", features = ["reqwest", "debug-api"], default-features = false } +alloy-pubsub = { version = "2.3.0", default-features = false } +alloy-rpc-client = { version = "2.3.0", default-features = false } +alloy-rpc-types = { version = "2.3.0", features = ["eth"], default-features = false } +alloy-rpc-types-admin = { version = "2.3.0", default-features = false } +alloy-rpc-types-anvil = { version = "2.3.0", default-features = false } +alloy-rpc-types-beacon = { version = "2.3.0", default-features = false } +alloy-rpc-types-debug = { version = "2.3.0", default-features = false } +alloy-rpc-types-engine = { version = "2.3.0", default-features = false } +alloy-rpc-types-eth = { version = "2.3.0", default-features = false } +alloy-rpc-types-mev = { version = "2.3.0", default-features = false } +alloy-rpc-types-trace = { version = "2.3.0", default-features = false } +alloy-rpc-types-txpool = { version = "2.3.0", default-features = false } +alloy-serde = { version = "2.3.0", default-features = false } +alloy-signer = { version = "2.3.0", default-features = false } +alloy-signer-local = { version = "2.3.0", default-features = false } +alloy-transport = { version = "2.3.0" } +alloy-transport-http = { version = "2.3.0", features = ["reqwest-rustls-tls"], default-features = false } +alloy-transport-ipc = { version = "2.3.0", default-features = false } +alloy-transport-ws = { version = "2.3.0", default-features = false } # misc either = { version = "1.16.0", default-features = false } @@ -553,7 +554,7 @@ quote = "1.0" # tokio tokio = { version = "1.52.3", default-features = false } tokio-stream = "0.1.11" -tokio-tungstenite = "0.28.0" +tokio-tungstenite = "0.29.0" tokio-util = { version = "0.7.4", features = ["codec"] } # async diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index 0449dc71299..d616349ceda 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -71,7 +71,7 @@ aquamarine.workspace = true clap = { workspace = true, features = ["derive", "env"] } [dev-dependencies] -alloy-node-bindings = "2.2.0" +alloy-node-bindings.workspace = true alloy-provider = { workspace = true, features = ["reqwest"] } alloy-rpc-types-eth.workspace = true backon.workspace = true From 10aa6a512ba7c4f5f01ff489b1f513da0745821f Mon Sep 17 00:00:00 2001 From: jihoon1125 <67624104+jihoon1125@users.noreply.github.com> Date: Mon, 3 Aug 2026 22:06:05 +0000 Subject: [PATCH 081/148] fix(rpc): include block numbers in pruned history error (#26550) --- crates/rpc/rpc-eth-types/src/error/mod.rs | 35 ++++++++++++++++++++--- crates/rpc/rpc/src/eth/filter.rs | 12 ++++++-- crates/rpc/rpc/src/trace.rs | 6 +++- 3 files changed, 46 insertions(+), 7 deletions(-) diff --git a/crates/rpc/rpc-eth-types/src/error/mod.rs b/crates/rpc/rpc-eth-types/src/error/mod.rs index 971a365aa39..94dcb7c5615 100644 --- a/crates/rpc/rpc-eth-types/src/error/mod.rs +++ b/crates/rpc/rpc-eth-types/src/error/mod.rs @@ -85,8 +85,13 @@ pub enum EthApiError { /// requested that has been pruned according to the node's data retention policy. /// /// See also - #[error("Pruned history unavailable")] - PrunedHistoryUnavailable, + #[error("pruned history unavailable: requested {requested}, earliest available {earliest_available}")] + PrunedHistoryUnavailable { + /// The requested block number + requested: u64, + /// The earliest block number that is still available + earliest_available: u64, + }, /// Receipts not found for block hash/number/tag #[error("receipts not found")] ReceiptsNotFound(BlockId), @@ -356,7 +361,9 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { internal_rpc_err(err.to_string()) } err @ EthApiError::TransactionInputError(_) => invalid_params_rpc_err(err.to_string()), - EthApiError::PrunedHistoryUnavailable => rpc_error_with_code(4444, error.to_string()), + EthApiError::PrunedHistoryUnavailable { .. } => { + rpc_error_with_code(4444, error.to_string()) + } EthApiError::Other(err) => err.to_rpc_error(), EthApiError::MuxTracerError(msg) => internal_rpc_err(msg.to_string()), EthApiError::BatchTxRecvError(err) => internal_rpc_err(err.to_string()), @@ -540,7 +547,9 @@ impl From for EthApiError { ProviderError::BlockNumberForTransactionIndexNotFound => Self::UnknownBlockOrTxIndex, ProviderError::FinalizedBlockNotFound => Self::HeaderNotFound(BlockId::finalized()), ProviderError::SafeBlockNotFound => Self::HeaderNotFound(BlockId::safe()), - ProviderError::BlockExpired { .. } => Self::PrunedHistoryUnavailable, + ProviderError::BlockExpired { requested, earliest_available } => { + Self::PrunedHistoryUnavailable { requested, earliest_available } + } err => Self::Internal(err.into()), } } @@ -1233,6 +1242,24 @@ mod tests { assert_eq!(err.message(), "block not found: finalized"); } + #[test] + fn pruned_history_error_reports_available_range() { + let err: EthApiError = + reth_errors::ProviderError::BlockExpired { requested: 5, earliest_available: 100 } + .into(); + assert!(matches!( + err, + EthApiError::PrunedHistoryUnavailable { requested: 5, earliest_available: 100 } + )); + + let err: jsonrpsee_types::error::ErrorObject<'static> = err.into(); + assert_eq!(err.code(), 4444); + assert_eq!( + err.message(), + "pruned history unavailable: requested 5, earliest available 100" + ); + } + #[test] fn receipts_not_found_message() { let err: jsonrpsee_types::error::ErrorObject<'static> = diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index fa1067506bb..e23dc4ded82 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -492,7 +492,11 @@ where // Check if the block has been pruned (EIP-4444) let earliest_block = self.provider().earliest_block_number()?; if block_number < earliest_block { - return Err(EthApiError::PrunedHistoryUnavailable.into()); + return Err(EthApiError::PrunedHistoryUnavailable { + requested: block_number, + earliest_available: earliest_block, + } + .into()); } let block_num_hash = BlockNumHash::new(block_number, block_hash); @@ -583,7 +587,11 @@ where // Check if the requested range overlaps with pruned history (EIP-4444) let earliest_block = self.provider().earliest_block_number()?; if from_block_number < earliest_block { - return Err(EthApiError::PrunedHistoryUnavailable.into()); + return Err(EthApiError::PrunedHistoryUnavailable { + requested: from_block_number, + earliest_available: earliest_block, + } + .into()); } self.get_logs_in_block_range(filter, from_block_number, to_block_number, limits) diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index 02dd965c735..bc8b88a350b 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -370,7 +370,11 @@ where let earliest_block = self.provider().earliest_block_number().map_err(Eth::Error::from_eth_err)?; if start < earliest_block { - return Err(EthApiError::PrunedHistoryUnavailable.into()); + return Err(EthApiError::PrunedHistoryUnavailable { + requested: start, + earliest_available: earliest_block, + } + .into()); } if start > end { From fca7532fcf7329b994852fe4e741581565e88d57 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Mon, 3 Aug 2026 18:07:22 -0500 Subject: [PATCH 082/148] feat(config): add bootnodes to reth.toml (#26551) Co-authored-by: Matthias Seitz --- crates/config/Cargo.toml | 3 +- crates/config/src/config.rs | 38 ++++++++++++++++++ crates/node/core/src/args/network.rs | 45 ++++++++++++++++++++-- docs/vocs/docs/pages/run/configuration.mdx | 14 +++++++ 4 files changed, 95 insertions(+), 5 deletions(-) diff --git a/crates/config/Cargo.toml b/crates/config/Cargo.toml index 6edabeab2b2..6351b83d7d8 100644 --- a/crates/config/Cargo.toml +++ b/crates/config/Cargo.toml @@ -12,6 +12,7 @@ workspace = true [dependencies] # reth +reth-network-peers.workspace = true reth-network-types.workspace = true reth-prune-types.workspace = true reth-stages-types.workspace = true @@ -33,6 +34,7 @@ serde = [ "dep:serde", "dep:toml", "dep:humantime-serde", + "reth-network-peers/secp256k1", "reth-network-types/serde", "reth-prune-types/serde", "reth-stages-types/serde", @@ -42,5 +44,4 @@ serde = [ [dev-dependencies] tempfile.workspace = true -reth-network-peers.workspace = true alloy-primitives = { workspace = true, features = ["getrandom"] } diff --git a/crates/config/src/config.rs b/crates/config/src/config.rs index ad0e71b0c62..e0d28692579 100644 --- a/crates/config/src/config.rs +++ b/crates/config/src/config.rs @@ -1,4 +1,5 @@ //! Configuration files. +use reth_network_peers::TrustedPeer; use reth_network_types::{PeersConfig, SessionsConfig}; use reth_prune_types::{PruneModes, MINIMUM_UNWIND_SAFE_DISTANCE}; use reth_stages_types::ExecutionStageThresholds; @@ -20,6 +21,10 @@ pub const DEFAULT_BLOCK_INTERVAL: usize = 5; #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] #[cfg_attr(feature = "serde", serde(default))] pub struct Config { + /// Nodes to bootstrap P2P discovery with, as `enode://` URLs or `enr:` records. + /// + /// Takes precedence over the chain spec bootnodes, and is overridden by `--bootnodes`. + pub bootnodes: Vec, /// Configuration for each stage in the pipeline. pub stages: StageConfig, /// Configuration for pruning. @@ -1200,4 +1205,37 @@ connect_trusted_nodes_only = true assert!(conf.peers.trusted_nodes.contains(&node)); } } + + #[test] + fn test_bootnodes() { + let reth_toml = r#" + bootnodes = [ + "enode://0401e494dbd0c84c5c0f72adac5985d2f2525e08b68d448958aae218f5ac8198a80d1498e0ebec2ce38b1b18d6750f6e61a56b4614c5a6c6cf0981c39aed47dc@34.159.32.127:30303", + "enode://e9675164b5e17b9d9edf0cc2bd79e6b6f487200c74d1331c220abb5b8ee80c2eefbf18213989585e9d0960683e819542e11d4eefb5f2b4019e1e49f9fd8fff18@berav2-bootnode.staketab.org:30303", + "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8" + ] + "#; + + let conf: Config = toml::from_str(reth_toml).unwrap(); + assert_eq!(conf.bootnodes.len(), 3); + assert_eq!( + conf.bootnodes[1].host, + url::Host::::Domain("berav2-bootnode.staketab.org".to_string()) + ); + // the ENR omits the tcp key, so the udp port is used for the RLPx dial guess + assert_eq!( + conf.bootnodes[2], + TrustedPeer::from_str("enode://ca634cae0d49acb401d8a4c6b6fe8c55b70d115bf400769cc1400f3258cd31387574077f301b421bc84df7266c44e9e6d569fc56be00812904767bf5ccd1fc7f@127.0.0.1:30303").unwrap() + ); + + // bootnodes are written back as enode URLs, including the ones read as ENRs + let serialized = toml::to_string(&conf).unwrap(); + assert_eq!(toml::from_str::(&serialized).unwrap(), conf); + } + + #[test] + fn test_bootnodes_default_empty() { + let conf: Config = toml::from_str("").unwrap(); + assert!(conf.bootnodes.is_empty()); + } } diff --git a/crates/node/core/src/args/network.rs b/crates/node/core/src/args/network.rs index d45bebe6859..fc2115f2dd5 100644 --- a/crates/node/core/src/args/network.rs +++ b/crates/node/core/src/args/network.rs @@ -487,8 +487,8 @@ impl NetworkArgs { /// Returns the resolved bootnodes if any are provided. pub fn resolved_bootnodes(&self) -> Option> { - self.bootnodes.clone().map(|bootnodes| { - bootnodes.into_iter().filter_map(|node| node.resolve_blocking().ok()).collect() + self.bootnodes.as_deref().map(|bootnodes| { + bootnodes.iter().filter_map(|node| node.resolve_blocking().ok()).collect() }) } @@ -546,8 +546,9 @@ impl NetworkArgs { /// Configured Bootnodes are prioritized, if unset, the chain spec bootnodes are used /// Priority order for bootnodes configuration: /// 1. --bootnodes flag - /// 2. Network preset flags (e.g. --holesky) - /// 3. default to mainnet nodes + /// 2. `bootnodes` in the config file + /// 3. Network preset flags (e.g. --holesky) + /// 4. default to mainnet nodes pub fn network_config( &self, config: &Config, @@ -565,6 +566,15 @@ impl NetworkArgs { let discovery_addr = self.resolved_discovery_addr(listener_addr); let chain_bootnodes = self .resolved_bootnodes() + .or_else(|| { + (!config.bootnodes.is_empty()).then(|| { + config + .bootnodes + .iter() + .filter_map(|node| node.resolve_blocking().ok()) + .collect() + }) + }) .unwrap_or_else(|| chain_spec.bootnodes().unwrap_or_else(mainnet_nodes)); let peers_file = self.peers_file.clone().unwrap_or(default_peers_file); @@ -1629,4 +1639,31 @@ mod tests { // Cleanup let _ = fs::remove_file(&peers_file); } + + #[test] + fn network_config_prefers_cli_bootnodes_over_config_file() { + let enr = "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8"; + let enode = "enode://6f8a80d14311c39f35f516fa664deaaaa13e85b2f7493f37f6144d86991ec012937307647bd3b9a82abe2974e1407241d54947bbb39763a4cac9f77166ad92a0@10.3.58.6:30303"; + let config = Config { bootnodes: vec![enr.parse().unwrap()], ..Default::default() }; + let secret_key = SecretKey::from_byte_array(&[1u8; 32]).unwrap(); + + let boot_nodes = |args: &NetworkArgs| { + args.network_config::( + &config, + MAINNET.clone(), + secret_key, + PathBuf::from("peers.json"), + Runtime::test(), + ) + .boot_nodes_iter() + .cloned() + .collect::>() + }; + + let args = NetworkArgs { no_persist_peers: true, ..Default::default() }; + assert_eq!(boot_nodes(&args), vec![enr.parse::().unwrap()]); + + let args = NetworkArgs { bootnodes: Some(vec![enode.parse().unwrap()]), ..args }; + assert_eq!(boot_nodes(&args), vec![enode.parse::().unwrap()]); + } } diff --git a/docs/vocs/docs/pages/run/configuration.mdx b/docs/vocs/docs/pages/run/configuration.mdx index 64a67e0b82c..d922544a77d 100644 --- a/docs/vocs/docs/pages/run/configuration.mdx +++ b/docs/vocs/docs/pages/run/configuration.mdx @@ -14,6 +14,7 @@ The default data directory is platform dependent: The configuration file contains the following sections: +- [`bootnodes`](#the-bootnodes-key) -- Nodes to bootstrap P2P discovery with - [`[stages]`](#the-stages-section) -- Configuration of the individual sync stages - [`era`](#era) - [`headers`](#headers) @@ -36,6 +37,19 @@ The configuration file contains the following sections: - [`[prune]`](#the-prune-section) - [`[static_files]`](#the-static_files-section) +## The `bootnodes` key + +The nodes reth bootstraps P2P discovery with, as `enode://` URLs or `enr:` records. Entries may use a domain name instead of an IP address, which is resolved on startup. + +This takes precedence over the bootnodes of the chain being run, and is overridden by `--bootnodes`. + +```toml +bootnodes = [ + "enode://d860a01f9722d78051619d1e2351aba3f43f943f6f00718d1b9baa4101932a1f5011f16bb2b1bb35db20d6fe28fa0bf09636d26a87d31de9ec6203eeedb1f666@18.138.108.67:30303", + "enr:-IS4QHCYrYZbAKWCBRlAy5zzaDZXJBGkcnh4MHcBFZntXNFrdvJjX04jRzjzCBOonrkTfj499SZuOh8R33Ls8RRcy5wBgmlkgnY0gmlwhH8AAAGJc2VjcDI1NmsxoQPKY0yuDUmstAHYpMa2_oxVtw0RW_QAdpzBQA8yWM0xOIN1ZHCCdl8", +] +``` + ## The `[stages]` section The stages section is used to configure how individual stages in reth behave, which has a direct impact on resource utilization and sync speed. From 71e5cc8482b92a78a20cb306f345308598b5abda Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Tue, 4 Aug 2026 04:48:31 +0530 Subject: [PATCH 083/148] feat(txpool): track blob cell availability (#25463) Co-authored-by: Matthias Seitz --- crates/net/network/src/eth_requests.rs | 15 +- crates/rpc/rpc/src/eth/helpers/transaction.rs | 1 + crates/transaction-pool/src/blobstore/disk.rs | 56 ++++---- crates/transaction-pool/src/blobstore/mem.rs | 25 ++-- crates/transaction-pool/src/blobstore/mod.rs | 128 +++++++++++++++++- crates/transaction-pool/src/blobstore/noop.rs | 13 +- crates/transaction-pool/src/noop.rs | 12 +- crates/transaction-pool/src/pool/mod.rs | 12 +- .../transaction-pool/src/test_utils/mock.rs | 4 +- crates/transaction-pool/src/traits.rs | 53 +++++++- crates/transaction-pool/src/validate/eth.rs | 9 +- crates/transaction-pool/src/validate/mod.rs | 7 +- 12 files changed, 245 insertions(+), 90 deletions(-) diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 9254314202b..51a31fc5396 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -858,7 +858,9 @@ mod tests { use reth_network_api::test_utils::PeersHandle; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_storage_api::noop::NoopProvider; - use reth_transaction_pool::blobstore::{BlobStoreCleanupStat, BlobStoreError}; + use reth_transaction_pool::blobstore::{ + BlobStoreCleanupStat, BlobStoreError, PooledBlobSidecar, + }; use std::sync::{ atomic::{AtomicUsize, Ordering}, Arc, @@ -872,18 +874,11 @@ mod tests { } impl BlobStore for CountingBlobStore { - fn insert( - &self, - _tx: B256, - _data: BlobTransactionSidecarVariant, - ) -> Result<(), BlobStoreError> { + fn insert(&self, _tx: B256, _data: PooledBlobSidecar) -> Result<(), BlobStoreError> { Ok(()) } - fn insert_all( - &self, - _txs: Vec<(B256, BlobTransactionSidecarVariant)>, - ) -> Result<(), BlobStoreError> { + fn insert_all(&self, _txs: Vec<(B256, PooledBlobSidecar)>) -> Result<(), BlobStoreError> { Ok(()) } diff --git a/crates/rpc/rpc/src/eth/helpers/transaction.rs b/crates/rpc/rpc/src/eth/helpers/transaction.rs index 05719ae028c..ecc1c431e1b 100644 --- a/crates/rpc/rpc/src/eth/helpers/transaction.rs +++ b/crates/rpc/rpc/src/eth/helpers/transaction.rs @@ -51,6 +51,7 @@ where Eip4844PoolTransactionError::MissingEip4844BlobSidecar, ))); }; + let sidecar = sidecar.into_sidecar(); let sidecar = match sidecar { BlobTransactionSidecarVariant::Eip4844(sidecar) => { diff --git a/crates/transaction-pool/src/blobstore/disk.rs b/crates/transaction-pool/src/blobstore/disk.rs index e5c081c0a3a..ed3ef441b3f 100644 --- a/crates/transaction-pool/src/blobstore/disk.rs +++ b/crates/transaction-pool/src/blobstore/disk.rs @@ -1,6 +1,8 @@ //! A simple diskstore for blobs -use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError, BlobStoreSize}; +use crate::blobstore::{ + BlobStore, BlobStoreCleanupStat, BlobStoreError, BlobStoreSize, PooledBlobSidecar, +}; use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, @@ -210,17 +212,15 @@ impl DiskFileBlobStore { } impl BlobStore for DiskFileBlobStore { - fn insert(&self, tx: B256, data: BlobTransactionSidecarVariant) -> Result<(), BlobStoreError> { - self.inner.insert_one(tx, data) + fn insert(&self, tx: B256, data: PooledBlobSidecar) -> Result<(), BlobStoreError> { + self.inner.insert_one(tx, data.into_sidecar()) } - fn insert_all( - &self, - txs: Vec<(B256, BlobTransactionSidecarVariant)>, - ) -> Result<(), BlobStoreError> { + fn insert_all(&self, txs: Vec<(B256, PooledBlobSidecar)>) -> Result<(), BlobStoreError> { if txs.is_empty() { return Ok(()) } + let txs = txs.into_iter().map(|(tx, data)| (tx, data.into_sidecar())).collect(); self.inner.insert_many(txs) } @@ -865,6 +865,12 @@ mod tests { .collect() } + fn wrapped_blobs( + blobs: Vec<(TxHash, BlobTransactionSidecarVariant)>, + ) -> Vec<(TxHash, PooledBlobSidecar)> { + blobs.into_iter().map(|(tx, blob)| (tx, blob.into())).collect() + } + fn eip7594_single_blob_sidecar() -> (BlobTransactionSidecarVariant, B256, BlobAndProofV2) { let blob = Blob::default(); let commitment = Bytes48::default(); @@ -885,7 +891,7 @@ mod tests { let blobs = rng_blobs(10); let all_hashes = blobs.iter().map(|(tx, _)| *tx).collect::>(); - store.insert_all(blobs.clone()).unwrap(); + store.insert_all(wrapped_blobs(blobs.clone())).unwrap(); // all cached for (tx, blob) in &blobs { @@ -922,7 +928,7 @@ mod tests { let (store, _dir) = tmp_store(); let (tx, blob) = rng_blobs(1).into_iter().next().unwrap(); - store.insert(tx, blob.clone()).unwrap(); + store.insert(tx, blob.clone().into()).unwrap(); assert!(store.is_cached(&tx)); let retrieved_blob = store.get(tx).unwrap().map(Arc::unwrap_or_clone).unwrap(); @@ -934,7 +940,7 @@ mod tests { let (store, _dir) = tmp_store(); let (tx, blob) = rng_blobs(1).into_iter().next().unwrap(); - store.insert(tx, blob).unwrap(); + store.insert(tx, blob.into()).unwrap(); assert!(store.is_cached(&tx)); store.delete(tx).unwrap(); @@ -958,7 +964,7 @@ mod tests { let blobs = rng_blobs(5); let txs = blobs.iter().map(|(tx, _)| *tx).collect::>(); - store.insert_all(blobs.clone()).unwrap(); + store.insert_all(wrapped_blobs(blobs.clone())).unwrap(); for (tx, _) in &blobs { assert!(store.is_cached(tx)); @@ -986,7 +992,7 @@ mod tests { let blobs = rng_blobs(3); let txs = blobs.iter().map(|(tx, _)| *tx).collect::>(); - store.insert_all(blobs.clone()).unwrap(); + store.insert_all(wrapped_blobs(blobs.clone())).unwrap(); let retrieved_blobs = store.get_all(txs.clone()).unwrap(); for (tx, blob) in retrieved_blobs { @@ -1003,7 +1009,7 @@ mod tests { let blobs = rng_blobs(3); let txs = blobs.iter().map(|(tx, _)| *tx).collect::>(); - store.insert_all(blobs.clone()).unwrap(); + store.insert_all(wrapped_blobs(blobs.clone())).unwrap(); let retrieved_blobs = store.get_exact(txs).unwrap(); for (retrieved_blob, (_, original_blob)) in retrieved_blobs.into_iter().zip(blobs) { @@ -1017,7 +1023,7 @@ mod tests { let blobs = rng_blobs(2); let txs = blobs.iter().map(|(tx, _)| *tx).collect::>(); - store.insert_all(blobs).unwrap(); + store.insert_all(wrapped_blobs(blobs)).unwrap(); // Try to get a blob that was never inserted let missing_tx = TxHash::random(); @@ -1031,7 +1037,7 @@ mod tests { assert_eq!(store.data_size_hint(), Some(0)); let blobs = rng_blobs(2); - store.insert_all(blobs).unwrap(); + store.insert_all(wrapped_blobs(blobs)).unwrap(); assert!(store.data_size_hint().unwrap() > 0); } @@ -1041,7 +1047,7 @@ mod tests { let blobs = rng_blobs(3); let txs = blobs.iter().map(|(tx, _)| *tx).collect::>(); - store.insert_all(blobs).unwrap(); + store.insert_all(wrapped_blobs(blobs)).unwrap(); store.delete_all(txs).unwrap(); let stat = store.cleanup(); @@ -1054,7 +1060,7 @@ mod tests { let (store, _dir) = tmp_store(); let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar(); - store.insert(TxHash::random(), sidecar).unwrap(); + store.insert(TxHash::random(), sidecar.into()).unwrap(); assert_ne!(versioned_hash, B256::ZERO); @@ -1071,7 +1077,7 @@ mod tests { let (store, _dir) = tmp_store(); let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); - store.insert(TxHash::random(), sidecar).unwrap(); + store.insert(TxHash::random(), sidecar.into()).unwrap(); let request = vec![B256::ZERO, versioned_hash, versioned_hash]; assert_eq!(store.has_versioned_hashes(&request).unwrap(), vec![false, true, true]); @@ -1082,7 +1088,7 @@ mod tests { let (store, _dir) = tmp_store(); let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); - store.insert(TxHash::random(), sidecar).unwrap(); + store.insert(TxHash::random(), sidecar.into()).unwrap(); let indices_bitarray = B128::from((1u128 << 0) | (1u128 << 7)); let request = vec![versioned_hash, B256::ZERO]; @@ -1103,7 +1109,7 @@ mod tests { let (store, _dir) = tmp_store(); let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar(); - store.insert(TxHash::random(), sidecar).unwrap(); + store.insert(TxHash::random(), sidecar.into()).unwrap(); store.clear_cache(); let v3 = store.get_by_versioned_hashes_v3(&[versioned_hash]).unwrap(); @@ -1115,7 +1121,7 @@ mod tests { let (store, _dir) = tmp_store(); let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); - store.insert(TxHash::random(), sidecar).unwrap(); + store.insert(TxHash::random(), sidecar.into()).unwrap(); store.clear_cache(); assert_eq!(store.has_versioned_hashes(&[versioned_hash]).unwrap(), vec![true]); @@ -1127,7 +1133,7 @@ mod tests { let tx_hash = TxHash::random(); let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); - store.insert(tx_hash, sidecar).unwrap(); + store.insert(tx_hash, sidecar.into()).unwrap(); store.clear_cache(); store.delete(tx_hash).unwrap(); @@ -1141,7 +1147,7 @@ mod tests { let (store, _dir) = tmp_store(); let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); - store.insert(TxHash::random(), sidecar).unwrap(); + store.insert(TxHash::random(), sidecar.into()).unwrap(); store.clear_cache(); let v4 = store.get_by_versioned_hashes_v4(&[versioned_hash], B128::from(1u128)).unwrap(); @@ -1156,7 +1162,7 @@ mod tests { let tx_hash = TxHash::random(); let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); - store.insert(tx_hash, sidecar).unwrap(); + store.insert(tx_hash, sidecar.into()).unwrap(); let indices_bitarray = B128::from((1u128 << 0) | (1u128 << 7)); let expected = store @@ -1181,7 +1187,7 @@ mod tests { let blobs = rng_blobs(5); let all_hashes: Vec<_> = blobs.iter().map(|(tx, _)| *tx).collect(); - store.insert_all(blobs).unwrap(); + store.insert_all(wrapped_blobs(blobs)).unwrap(); store.clear_cache(); // Schedule blobs for deletion diff --git a/crates/transaction-pool/src/blobstore/mem.rs b/crates/transaction-pool/src/blobstore/mem.rs index de6ccad02d9..78637605c8e 100644 --- a/crates/transaction-pool/src/blobstore/mem.rs +++ b/crates/transaction-pool/src/blobstore/mem.rs @@ -1,4 +1,6 @@ -use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError, BlobStoreSize}; +use crate::blobstore::{ + BlobStore, BlobStoreCleanupStat, BlobStoreError, BlobStoreSize, PooledBlobSidecar, +}; use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, @@ -95,24 +97,21 @@ impl PartialEq for InMemoryBlobStoreInner { } impl BlobStore for InMemoryBlobStore { - fn insert(&self, tx: B256, data: BlobTransactionSidecarVariant) -> Result<(), BlobStoreError> { + fn insert(&self, tx: B256, data: PooledBlobSidecar) -> Result<(), BlobStoreError> { let mut store = self.inner.store.write(); - self.inner.size_tracker.add_size(insert_size(&mut store, tx, data)); + self.inner.size_tracker.add_size(insert_size(&mut store, tx, data.into_sidecar())); self.inner.size_tracker.update_len(store.len()); Ok(()) } - fn insert_all( - &self, - txs: Vec<(B256, BlobTransactionSidecarVariant)>, - ) -> Result<(), BlobStoreError> { + fn insert_all(&self, txs: Vec<(B256, PooledBlobSidecar)>) -> Result<(), BlobStoreError> { if txs.is_empty() { return Ok(()) } let mut store = self.inner.store.write(); let mut total_add = 0; for (tx, data) in txs { - let add = insert_size(&mut store, tx, data); + let add = insert_size(&mut store, tx, data.into_sidecar()); total_add += add; } self.inner.size_tracker.add_size(total_add); @@ -336,8 +335,8 @@ mod tests { let (eip7594_sidecar, eip7594_hash, _) = eip7594_single_blob_sidecar(); let (eip4844_sidecar, eip4844_hash) = eip4844_single_blob_sidecar(); - store.insert(B256::random(), eip7594_sidecar).unwrap(); - store.insert(B256::random(), eip4844_sidecar).unwrap(); + store.insert(B256::random(), eip7594_sidecar.into()).unwrap(); + store.insert(B256::random(), eip4844_sidecar.into()).unwrap(); let request = vec![eip7594_hash, B256::ZERO, eip4844_hash, eip7594_hash]; assert_eq!(store.has_versioned_hashes(&request).unwrap(), vec![true, false, true, true]); @@ -348,7 +347,7 @@ mod tests { let store = InMemoryBlobStore::default(); let (sidecar, versioned_hash, expected) = eip7594_single_blob_sidecar(); - store.insert(B256::random(), sidecar).unwrap(); + store.insert(B256::random(), sidecar.into()).unwrap(); assert_ne!(versioned_hash, B256::ZERO); @@ -365,7 +364,7 @@ mod tests { let store = InMemoryBlobStore::default(); let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); - store.insert(B256::random(), sidecar).unwrap(); + store.insert(B256::random(), sidecar.into()).unwrap(); let indices_bitarray = B128::from((1u128 << 0) | (1u128 << 7)); let request = vec![versioned_hash, B256::ZERO]; @@ -387,7 +386,7 @@ mod tests { let tx_hash = B256::random(); let (sidecar, versioned_hash, _) = eip7594_single_blob_sidecar(); - store.insert(tx_hash, sidecar).unwrap(); + store.insert(tx_hash, sidecar.into()).unwrap(); let indices_bitarray = B128::from((1u128 << 0) | (1u128 << 7)); let expected = store diff --git a/crates/transaction-pool/src/blobstore/mod.rs b/crates/transaction-pool/src/blobstore/mod.rs index 9b6c72f9a37..eaec381933d 100644 --- a/crates/transaction-pool/src/blobstore/mod.rs +++ b/crates/transaction-pool/src/blobstore/mod.rs @@ -2,7 +2,7 @@ use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, - eip7594::{BlobTransactionSidecarVariant, Cell}, + eip7594::{BlobCellMask, BlobTransactionSidecarVariant, Cell}, }; use alloy_primitives::{TxHash, B128, B256}; pub use converter::BlobSidecarConverter; @@ -11,8 +11,9 @@ pub use mem::InMemoryBlobStore; pub use noop::NoopBlobStore; use std::{ fmt, + ops::Deref, sync::{ - atomic::{AtomicUsize, Ordering}, + atomic::{AtomicU64, AtomicUsize, Ordering}, Arc, }, }; @@ -24,6 +25,98 @@ mod mem; mod noop; mod tracker; +/// Blob cell availability stored for a transaction. +/// +/// Bit `i` corresponds to cell index `i`. The two words are stored least-significant first: index +/// `0` contains cells `0..64` and index `1` contains cells `64..128`. +#[derive(Debug, Clone)] +pub struct BlobCellAvailability(Arc<[AtomicU64; 2]>); + +impl BlobCellAvailability { + const LOW_WORD: usize = 0; + const HIGH_WORD: usize = 1; + + /// Returns full availability for all blob cells. + pub fn full() -> Self { + Self(Arc::new([AtomicU64::new(u64::MAX), AtomicU64::new(u64::MAX)])) + } + + /// Returns a snapshot of the available cells. + /// + /// The two words are loaded independently. Future writers must only add availability bits so + /// that a concurrent snapshot can understate availability but never overstate it. + pub fn get(&self) -> BlobCellMask { + let low = self.0[Self::LOW_WORD].load(Ordering::Relaxed) as u128; + let high = self.0[Self::HIGH_WORD].load(Ordering::Relaxed) as u128; + BlobCellMask::from_bits((high << 64) | low) + } + + /// Returns true if all blob cells are available. + pub fn is_full(&self) -> bool { + self.get().bits() == u128::MAX + } +} + +impl PartialEq for BlobCellAvailability { + fn eq(&self, other: &Self) -> bool { + self.get() == other.get() + } +} + +impl Eq for BlobCellAvailability {} + +/// A blob sidecar paired with its shared cell availability. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct PooledBlobSidecar { + sidecar: BlobTransactionSidecarVariant, + availability: BlobCellAvailability, +} + +impl PooledBlobSidecar { + /// Creates a sidecar with the given shared cell availability. + pub const fn new( + sidecar: BlobTransactionSidecarVariant, + availability: BlobCellAvailability, + ) -> Self { + Self { sidecar, availability } + } + + /// Returns the wrapped sidecar. + pub const fn sidecar(&self) -> &BlobTransactionSidecarVariant { + &self.sidecar + } + + /// Returns whether this is an EIP-7594 sidecar. + pub const fn is_eip7594(&self) -> bool { + self.sidecar.is_eip7594() + } + + /// Returns the shared cell availability. + pub const fn availability(&self) -> &BlobCellAvailability { + &self.availability + } + + /// Consumes the wrapper and returns the sidecar. + pub fn into_sidecar(self) -> BlobTransactionSidecarVariant { + self.sidecar + } +} + +impl Deref for PooledBlobSidecar { + type Target = BlobTransactionSidecarVariant; + + fn deref(&self) -> &Self::Target { + &self.sidecar + } +} + +impl From for PooledBlobSidecar { + fn from(sidecar: BlobTransactionSidecarVariant) -> Self { + // TODO: Initialize this with the actual mask once sparse sidecars are supported. + Self::new(sidecar, BlobCellAvailability::full()) + } +} + /// A blob store that can be used to store blob data of EIP4844 transactions. /// /// This type is responsible for keeping track of blob data until it is no longer needed (after @@ -32,13 +125,10 @@ mod tracker; /// Note: this is Clone because it is expected to be wrapped in an Arc. pub trait BlobStore: fmt::Debug + Send + Sync + 'static { /// Inserts the blob sidecar into the store - fn insert(&self, tx: B256, data: BlobTransactionSidecarVariant) -> Result<(), BlobStoreError>; + fn insert(&self, tx: B256, data: PooledBlobSidecar) -> Result<(), BlobStoreError>; /// Inserts multiple blob sidecars into the store - fn insert_all( - &self, - txs: Vec<(B256, BlobTransactionSidecarVariant)>, - ) -> Result<(), BlobStoreError>; + fn insert_all(&self, txs: Vec<(B256, PooledBlobSidecar)>) -> Result<(), BlobStoreError>; /// Deletes the blob sidecar from the store fn delete(&self, tx: B256) -> Result<(), BlobStoreError>; @@ -239,9 +329,33 @@ pub struct BlobStoreCleanupStat { #[cfg(test)] mod tests { use super::*; + use alloy_eips::{eip4844::BlobTransactionSidecar, eip7594::BlobTransactionSidecarEip7594}; #[expect(dead_code)] struct DynStore { store: Box, } + + #[test] + fn pooled_blob_sidecar_defaults_to_full_availability() { + let sidecars = [ + BlobTransactionSidecarVariant::Eip4844(BlobTransactionSidecar::default()), + BlobTransactionSidecarVariant::Eip7594(BlobTransactionSidecarEip7594::default()), + ]; + + for sidecar in sidecars { + assert!(PooledBlobSidecar::from(sidecar).availability().is_full()); + } + } + + #[test] + fn blob_cell_availability_uses_cell_index_bit_order() { + let availability = + BlobCellAvailability(Arc::new([AtomicU64::new(1), AtomicU64::new(1 << 1)])); + + let mask = availability.get(); + assert!(mask.contains(0)); + assert!(mask.contains(65)); + assert_eq!(mask.count(), 2); + } } diff --git a/crates/transaction-pool/src/blobstore/noop.rs b/crates/transaction-pool/src/blobstore/noop.rs index c9220ec4422..80e9ae412fa 100644 --- a/crates/transaction-pool/src/blobstore/noop.rs +++ b/crates/transaction-pool/src/blobstore/noop.rs @@ -1,4 +1,4 @@ -use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError}; +use crate::blobstore::{BlobStore, BlobStoreCleanupStat, BlobStoreError, PooledBlobSidecar}; use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobTransactionSidecarVariant, Cell}, @@ -12,18 +12,11 @@ use std::sync::Arc; pub struct NoopBlobStore; impl BlobStore for NoopBlobStore { - fn insert( - &self, - _tx: B256, - _data: BlobTransactionSidecarVariant, - ) -> Result<(), BlobStoreError> { + fn insert(&self, _tx: B256, _data: PooledBlobSidecar) -> Result<(), BlobStoreError> { Ok(()) } - fn insert_all( - &self, - _txs: Vec<(B256, BlobTransactionSidecarVariant)>, - ) -> Result<(), BlobStoreError> { + fn insert_all(&self, _txs: Vec<(B256, PooledBlobSidecar)>) -> Result<(), BlobStoreError> { Ok(()) } diff --git a/crates/transaction-pool/src/noop.rs b/crates/transaction-pool/src/noop.rs index 5ce4ec6ae0d..cca97f611b4 100644 --- a/crates/transaction-pool/src/noop.rs +++ b/crates/transaction-pool/src/noop.rs @@ -10,9 +10,10 @@ use crate::{ traits::{BestTransactionsAttributes, GetPooledTransactionLimit, NewBlobSidecar}, validate::ValidTransaction, AddedTransactionOutcome, AllPoolTransactions, AllTransactionsEvents, BestTransactions, - BlockInfo, EthPoolTransaction, EthPooledTransaction, NewTransactionEvent, PoolResult, PoolSize, - PoolTransaction, PropagatedTransactions, TransactionEvents, TransactionOrigin, TransactionPool, - TransactionValidationOutcome, TransactionValidator, ValidPoolTransaction, + BlockInfo, EthBlobTransactionSidecar, EthPoolTransaction, EthPooledTransaction, + NewTransactionEvent, PoolResult, PoolSize, PoolTransaction, PropagatedTransactions, + TransactionEvents, TransactionOrigin, TransactionPool, TransactionValidationOutcome, + TransactionValidator, ValidPoolTransaction, }; use alloy_eips::{ eip1559::ETHEREUM_BLOCK_GAS_LIMIT_30M, @@ -431,7 +432,10 @@ impl TransactionValidator for MockTransactionValidator InvalidPoolTransactionError::Underpriced, ); } - let maybe_sidecar = transaction.take_blob().maybe_sidecar().cloned(); + let maybe_sidecar = match transaction.take_blob() { + EthBlobTransactionSidecar::Present(sidecar) => Some(sidecar), + _ => None, + }; // we return `balance: U256::MAX` to simulate a valid transaction which will never go into // overdraft TransactionValidationOutcome::Valid { diff --git a/crates/transaction-pool/src/pool/mod.rs b/crates/transaction-pool/src/pool/mod.rs index d098d3e89a4..6d1139a4c18 100644 --- a/crates/transaction-pool/src/pool/mod.rs +++ b/crates/transaction-pool/src/pool/mod.rs @@ -66,7 +66,7 @@ //! category (2.) and become pending. use crate::{ - blobstore::BlobStore, + blobstore::{BlobStore, PooledBlobSidecar}, error::{PoolError, PoolErrorKind, PoolResult}, identifier::{SenderId, SenderIdentifiers, TransactionId}, metrics::BlobStoreMetrics, @@ -1300,7 +1300,7 @@ where } /// Inserts a blob transaction into the blob store - fn insert_blob(&self, hash: TxHash, blob: BlobTransactionSidecarVariant) { + fn insert_blob(&self, hash: TxHash, blob: PooledBlobSidecar) { debug!(target: "txpool", "[{:?}] storing blob sidecar", hash); if let Err(err) = self.blob_store.insert(hash, blob) { warn!(target: "txpool", %err, "[{:?}] failed to insert blob", hash); @@ -1362,7 +1362,7 @@ struct AddedTransactionMeta { /// The transaction that was added to the pool added: AddedTransaction, /// Optional blob sidecar for EIP-4844 transactions - blob_sidecar: Option, + blob_sidecar: Option, } /// Tracks an added transaction and all graph changes caused by adding it. @@ -1666,7 +1666,7 @@ impl OnNewCanonicalStateOutcome { #[cfg(test)] mod tests { use crate::{ - blobstore::{BlobStore, InMemoryBlobStore}, + blobstore::{BlobStore, InMemoryBlobStore, PooledBlobSidecar}, identifier::SenderId, test_utils::{MockTransaction, TestPoolBuilder}, validate::ValidTransaction, @@ -1731,7 +1731,7 @@ mod tests { // Insert the sidecar into the blob store if the current index is within the blob limit. if n < blob_limit.max_txs { - blob_store.insert(*tx.get_hash(), sidecar.clone()).unwrap(); + blob_store.insert(*tx.get_hash(), sidecar.clone().into()).unwrap(); } // Add the transaction to the pool with external origin and valid outcome. @@ -1743,7 +1743,7 @@ mod tests { bytecode_hash: None, transaction: ValidTransaction::ValidWithSidecar { transaction: tx, - sidecar: sidecar.clone(), + sidecar: PooledBlobSidecar::from(sidecar.clone()), }, propagate: true, authorities: None, diff --git a/crates/transaction-pool/src/test_utils/mock.rs b/crates/transaction-pool/src/test_utils/mock.rs index 33c7656e49d..188e46499a6 100644 --- a/crates/transaction-pool/src/test_utils/mock.rs +++ b/crates/transaction-pool/src/test_utils/mock.rs @@ -917,7 +917,9 @@ impl alloy_consensus::Transaction for MockTransaction { impl EthPoolTransaction for MockTransaction { fn take_blob(&mut self) -> EthBlobTransactionSidecar { match self { - Self::Eip4844 { sidecar, .. } => EthBlobTransactionSidecar::Present(sidecar.clone()), + Self::Eip4844 { sidecar, .. } => { + EthBlobTransactionSidecar::Present(sidecar.clone().into()) + } _ => EthBlobTransactionSidecar::None, } } diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 2507d200819..59e9a062bcf 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -51,7 +51,7 @@ //! - Conversion from consensus to pooled always fails use crate::{ - blobstore::{BlobStore, BlobStoreError}, + blobstore::{BlobCellAvailability, BlobStore, BlobStoreError, PooledBlobSidecar}, error::{InvalidPoolTransactionError, PoolError, PoolResult, RawPoolTransactionError}, pool::{ state::SubPool, BestTransactionFilter, NewTransactionEvent, TransactionEvents, @@ -1489,6 +1489,11 @@ pub trait EthPoolTransaction: PoolTransaction { /// Extracts the blob sidecar from the transaction. fn take_blob(&mut self) -> EthBlobTransactionSidecar; + /// Returns the shared blob cell availability, if this is a blob transaction. + fn blob_cell_availability(&self) -> Option<&BlobCellAvailability> { + None + } + /// A specialization for the EIP-4844 transaction type. /// Tries to reattach the blob sidecar to the transaction. /// @@ -1523,6 +1528,7 @@ pub trait EthPoolTransaction: PoolTransaction { /// - `cost`: Pre-calculated max cost (gas * price + value + blob costs) /// - `encoded_length`: Cached RLP encoding length for size limits /// - `blob_sidecar`: Blob data state (None/Missing/Present) +/// - `blob_cell_availability`: Cached blob cell availability for eth/72 announcements /// /// This avoids recalculating these values repeatedly during pool operations. #[derive(Debug, Clone, PartialEq, Eq)] @@ -1542,6 +1548,11 @@ pub struct EthPooledTransaction { /// The blob side car for this transaction pub blob_sidecar: EthBlobTransactionSidecar, + + /// Cached blob cell availability for this transaction. + /// + /// This is shared with the blob sidecar so that availability updates are reflected here. + blob_cell_availability: Option, } impl EthPooledTransaction { @@ -1550,6 +1561,7 @@ impl EthPooledTransaction { /// Caution: In case of blob transactions, this marks the blob sidecar as /// [`EthBlobTransactionSidecar::Missing`] pub fn new(transaction: Recovered, encoded_length: usize) -> Self { + let mut blob_cell_availability = None; let mut blob_sidecar = EthBlobTransactionSidecar::None; let gas_cost = U256::from(transaction.max_fee_per_gas()) @@ -1568,15 +1580,22 @@ impl EthPooledTransaction { // because the blob sidecar is not included in this transaction variant, mark it as // missing blob_sidecar = EthBlobTransactionSidecar::Missing; + // TODO: Initialize this with the actual mask once sparse sidecars are supported. + blob_cell_availability = Some(BlobCellAvailability::full()); } - Self { transaction, cost, encoded_length, blob_sidecar } + Self { transaction, cost, encoded_length, blob_sidecar, blob_cell_availability } } /// Return the reference to the underlying transaction. pub const fn transaction(&self) -> &Recovered { &self.transaction } + + /// Returns the shared blob cell availability, if this is a blob transaction. + pub const fn blob_cell_availability(&self) -> Option<&BlobCellAvailability> { + self.blob_cell_availability.as_ref() + } } impl PoolTransaction for EthPooledTransaction { @@ -1610,7 +1629,11 @@ impl PoolTransaction for EthPooledTransaction { let tx = TransactionSigned::from(tx); let tx = Recovered::new_unchecked(tx, signer); let mut pooled = Self::new(tx, encoded_length); - pooled.blob_sidecar = EthBlobTransactionSidecar::Present(blob); + if let Some(availability) = pooled.blob_cell_availability.clone() { + pooled.blob_sidecar = EthBlobTransactionSidecar::Present( + PooledBlobSidecar::new(blob, availability), + ); + } pooled } tx => { @@ -1743,6 +1766,10 @@ impl EthPoolTransaction for EthPooledTransaction { } } + fn blob_cell_availability(&self) -> Option<&BlobCellAvailability> { + Self::blob_cell_availability(self) + } + fn try_into_pooled_eip4844( self, sidecar: Arc, @@ -1803,14 +1830,14 @@ pub enum EthBlobTransactionSidecar { /// /// The sidecar is required for validating the transaction but is not included /// in blocks (only the blob hashes are included in the consensus format). - Present(BlobTransactionSidecarVariant), + Present(PooledBlobSidecar), } impl EthBlobTransactionSidecar { /// Returns the blob sidecar if it is present pub const fn maybe_sidecar(&self) -> Option<&BlobTransactionSidecarVariant> { match self { - Self::Present(sidecar) => Some(sidecar), + Self::Present(sidecar) => Some(sidecar.sidecar()), _ => None, } } @@ -1941,11 +1968,12 @@ impl Stream for NewSubpoolTransactionStream { #[cfg(test)] mod tests { use super::*; + use crate::blobstore::BlobCellAvailability; use alloy_consensus::{ EthereumTxEnvelope, SignableTransaction, TxEip1559, TxEip2930, TxEip4844, TxEip7702, TxEnvelope, TxLegacy, }; - use alloy_eips::eip4844::DATA_GAS_PER_BLOB; + use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7594::BlobCellMask}; use alloy_primitives::Signature; #[test] @@ -2004,6 +2032,8 @@ mod tests { assert_eq!(pooled_tx.transaction, transaction); assert_eq!(pooled_tx.encoded_length, 200); assert_eq!(pooled_tx.blob_sidecar, EthBlobTransactionSidecar::None); + assert!(pooled_tx.blob_cell_availability.is_none()); + assert_eq!(pooled_tx.blob_cell_availability().map(BlobCellAvailability::get), None); assert_eq!(pooled_tx.cost, U256::from(100) + U256::from(10 * 1000)); } @@ -2026,6 +2056,8 @@ mod tests { assert_eq!(pooled_tx.transaction, transaction); assert_eq!(pooled_tx.encoded_length, 200); assert_eq!(pooled_tx.blob_sidecar, EthBlobTransactionSidecar::None); + assert!(pooled_tx.blob_cell_availability.is_none()); + assert_eq!(pooled_tx.blob_cell_availability().map(BlobCellAvailability::get), None); assert_eq!(pooled_tx.cost, expected_cost); } @@ -2048,6 +2080,8 @@ mod tests { assert_eq!(pooled_tx.transaction, transaction); assert_eq!(pooled_tx.encoded_length, 200); assert_eq!(pooled_tx.blob_sidecar, EthBlobTransactionSidecar::None); + assert!(pooled_tx.blob_cell_availability.is_none()); + assert_eq!(pooled_tx.blob_cell_availability().map(BlobCellAvailability::get), None); assert_eq!(pooled_tx.cost, U256::from(100) + U256::from(10 * 1000)); } @@ -2072,6 +2106,11 @@ mod tests { assert_eq!(pooled_tx.transaction, transaction); assert_eq!(pooled_tx.encoded_length, 300); assert_eq!(pooled_tx.blob_sidecar, EthBlobTransactionSidecar::Missing); + assert!(pooled_tx.blob_cell_availability.is_some()); + assert_eq!( + pooled_tx.blob_cell_availability().map(BlobCellAvailability::get), + Some(BlobCellMask::from_bits(u128::MAX)) + ); let expected_cost = U256::from(100) + U256::from(10 * 1000) + U256::from(5 * DATA_GAS_PER_BLOB); assert_eq!(pooled_tx.cost, expected_cost); @@ -2096,6 +2135,8 @@ mod tests { assert_eq!(pooled_tx.transaction, transaction); assert_eq!(pooled_tx.encoded_length, 200); assert_eq!(pooled_tx.blob_sidecar, EthBlobTransactionSidecar::None); + assert!(pooled_tx.blob_cell_availability.is_none()); + assert_eq!(pooled_tx.blob_cell_availability().map(BlobCellAvailability::get), None); assert_eq!(pooled_tx.cost, U256::from(100) + U256::from(10 * 1000)); } diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index c7fc15fd641..eeb8f41dadc 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -2,16 +2,15 @@ use super::constants::DEFAULT_MAX_TX_INPUT_BYTES; use crate::{ - blobstore::BlobStore, + blobstore::{BlobStore, PooledBlobSidecar}, error::{ Eip4844PoolTransactionError, Eip7702PoolTransactionError, InvalidPoolTransactionError, }, metrics::TxPoolValidationMetrics, traits::TransactionOrigin, validate::ValidTransaction, - Address, BlobTransactionSidecarVariant, EthBlobTransactionSidecar, EthPoolTransaction, - LocalTransactionConfig, TransactionValidationOutcome, TransactionValidationTaskExecutor, - TransactionValidator, + Address, EthBlobTransactionSidecar, EthPoolTransaction, LocalTransactionConfig, + TransactionValidationOutcome, TransactionValidationTaskExecutor, TransactionValidator, }; use alloy_consensus::{ @@ -795,7 +794,7 @@ where pub fn validate_eip4844( &self, transaction: &mut Tx, - ) -> Result, InvalidPoolTransactionError> { + ) -> Result, InvalidPoolTransactionError> { let mut maybe_blob_sidecar = None; // heavy blob tx validation diff --git a/crates/transaction-pool/src/validate/mod.rs b/crates/transaction-pool/src/validate/mod.rs index dce498ccf4b..66d9b250ba4 100644 --- a/crates/transaction-pool/src/validate/mod.rs +++ b/crates/transaction-pool/src/validate/mod.rs @@ -1,12 +1,13 @@ //! Transaction validation abstractions. use crate::{ + blobstore::PooledBlobSidecar, error::InvalidPoolTransactionError, identifier::{SenderId, TransactionId}, traits::{PoolTransaction, TransactionOrigin}, PriceBumpConfig, }; -use alloy_eips::{eip7594::BlobTransactionSidecarVariant, eip7702::SignedAuthorization}; +use alloy_eips::eip7702::SignedAuthorization; use alloy_primitives::{Address, TxHash, B256, U256}; use futures_util::future::Either; use reth_primitives_traits::{Block, Recovered, SealedBlock}; @@ -116,13 +117,13 @@ pub enum ValidTransaction { /// The valid EIP-4844 transaction. transaction: T, /// The extracted sidecar of that transaction - sidecar: BlobTransactionSidecarVariant, + sidecar: PooledBlobSidecar, }, } impl ValidTransaction { /// Creates a new valid transaction with an optional sidecar. - pub fn new(transaction: T, sidecar: Option) -> Self { + pub fn new(transaction: T, sidecar: Option) -> Self { if let Some(sidecar) = sidecar { Self::ValidWithSidecar { transaction, sidecar } } else { From 3e9b1cb1f49417d98ea3bd372b228cc9ea5304cf Mon Sep 17 00:00:00 2001 From: Debjit Bhowal Date: Tue, 4 Aug 2026 07:26:56 +0530 Subject: [PATCH 084/148] feat: make testing rpc handler generic over types (#26547) Co-authored-by: Matthias Seitz --- crates/rpc/rpc/Cargo.toml | 2 +- crates/rpc/rpc/src/testing.rs | 73 +++++++++++++++++++++++------------ 2 files changed, 50 insertions(+), 25 deletions(-) diff --git a/crates/rpc/rpc/Cargo.toml b/crates/rpc/rpc/Cargo.toml index b6035a2faeb..fc729b3fb70 100644 --- a/crates/rpc/rpc/Cargo.toml +++ b/crates/rpc/rpc/Cargo.toml @@ -38,7 +38,6 @@ reth-rpc-server-types.workspace = true reth-network-types.workspace = true reth-consensus.workspace = true reth-consensus-common.workspace = true -reth-ethereum-primitives.workspace = true reth-ethereum-engine-primitives.workspace = true reth-node-api.workspace = true reth-payload-primitives.workspace = true @@ -94,6 +93,7 @@ itertools.workspace = true [dev-dependencies] reth-testing-utils.workspace = true +reth-ethereum-primitives.workspace = true reth-transaction-pool = { workspace = true, features = ["test-utils"] } reth-provider = { workspace = true, features = ["test-utils"] } reth-db-api.workspace = true diff --git a/crates/rpc/rpc/src/testing.rs b/crates/rpc/rpc/src/testing.rs index 3783a415e55..ad4ae5e14d2 100644 --- a/crates/rpc/rpc/src/testing.rs +++ b/crates/rpc/rpc/src/testing.rs @@ -14,7 +14,7 @@ //! **Warning:** This namespace allows building arbitrary blocks. Never expose it //! on public-facing RPC endpoints without proper authentication. -use alloy_consensus::{Header, Transaction}; +use alloy_consensus::Transaction; use alloy_eips::{eip1559::calculate_block_gas_limit, eip2718::Decodable2718}; use alloy_evm::{Evm, RecoveredTx}; use alloy_primitives::{ @@ -22,7 +22,10 @@ use alloy_primitives::{ Address, Bytes, B256, U256, }; use alloy_rlp::Encodable; -use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV5, ForkchoiceState, PayloadAttributes}; +use alloy_rpc_types_engine::{ + BlobsBundleV2, ExecutionData, ExecutionPayloadEnvelopeV5, ExecutionPayloadSidecar, + ExecutionPayloadV3, ForkchoiceState, PayloadAttributes, PraguePayloadFields, +}; use async_trait::async_trait; use jsonrpsee::core::RpcResult; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; @@ -30,12 +33,11 @@ use reth_consensus_common::validation::MAX_RLP_BLOCK_SIZE; use reth_engine_primitives::ConsensusEngineHandle; use reth_errors::RethError; use reth_ethereum_engine_primitives::EthBuiltPayload; -use reth_ethereum_primitives::EthPrimitives; use reth_evm::{execute::BlockBuilder, ConfigureEvm, NextBlockEnvAttributes}; -use reth_payload_primitives::PayloadTypes; +use reth_payload_primitives::{BuiltPayload, PayloadTypes}; use reth_primitives_traits::{ transaction::{recover::try_recover_signers, signed::RecoveryError}, - AlloyBlockHeader as BlockTrait, TxTy, + AlloyBlockHeader as BlockTrait, Block as _, HeaderTy, TxTy, }; use reth_revm::{database::StateProviderDatabase, db::State}; use reth_rpc_api::{TestingApiServer, TestingBuildBlockRequestV1}; @@ -100,23 +102,24 @@ impl TestingApi { impl TestingApi where - Payload: PayloadTypes, - Payload::ExecutionData: From, + Payload: PayloadTypes< + ExecutionData = ExecutionData, + BuiltPayload: BuiltPayload, + >, Eth: Call< - Provider: BlockReader
- + BlockReaderIdExt
+ Provider: BlockReader
> + + BlockReaderIdExt
> + ChainSpecProvider, Pool: TransactionPool>>, >, - Evm: ConfigureEvm - + 'static, + Evm: ConfigureEvm + 'static, { async fn build_payload_v1( &self, request: TestingBuildBlockRequestV1, skip_invalid_transactions: bool, use_pool_transactions: bool, - ) -> Result { + ) -> Result, Eth::Error> { let evm_config = self.evm_config.clone(); let desired_gas_limit = self.desired_gas_limit; let gas_limit_override = self.gas_limit_override; @@ -282,11 +285,22 @@ where request: TestingBuildBlockRequestV1, use_pool_transactions: bool, ) -> Result { - self.build_payload_v1(request, self.skip_invalid_transactions, use_pool_transactions) - .await? - .try_into_v5() - .map_err(RethError::other) - .map_err(Eth::Error::from_eth_err) + let payload = self + .build_payload_v1(request, self.skip_invalid_transactions, use_pool_transactions) + .await?; + let fees = payload.fees(); + let requests = payload.requests().unwrap_or_default(); + let block = Arc::unwrap_or_clone(payload.into_block_arc()); + let block_hash = block.hash(); + let block = block.into_block().into_ethereum_block(); + + Ok(ExecutionPayloadEnvelopeV5 { + execution_payload: ExecutionPayloadV3::from_block_unchecked(block_hash, &block), + block_value: fees, + blobs_bundle: BlobsBundleV2::empty(), + should_override_builder: false, + execution_requests: requests, + }) } async fn commit_block_v1( @@ -331,7 +345,17 @@ where .await?; let block_hash = payload.block().hash(); - let execution_data: Payload::ExecutionData = payload.into(); + let requests = payload.requests(); + let block_access_list = payload.block_access_list().cloned(); + let block = Arc::unwrap_or_clone(payload.into_block_arc()).into_sealed_block(); + let execution_data = Payload::block_to_payload(block, block_access_list); + let execution_data = match (requests, execution_data.sidecar.cancun()) { + (Some(requests), Some(cancun)) => ExecutionData::new( + execution_data.payload, + ExecutionPayloadSidecar::v4(cancun.clone(), PraguePayloadFields::new(requests)), + ), + _ => execution_data, + }; let status = self .engine_handle .new_payload(execution_data) @@ -372,16 +396,17 @@ where #[async_trait] impl TestingApiServer for TestingApi where - Payload: PayloadTypes, - Payload::ExecutionData: From, + Payload: PayloadTypes< + ExecutionData = ExecutionData, + BuiltPayload: BuiltPayload, + >, Eth: Call< - Provider: BlockReader
- + BlockReaderIdExt
+ Provider: BlockReader
> + + BlockReaderIdExt
> + ChainSpecProvider, Pool: TransactionPool>>, >, - Evm: ConfigureEvm - + 'static, + Evm: ConfigureEvm + 'static, { /// Handles `testing_buildBlockV1` by gating concurrency via a semaphore and offloading heavy /// work to the blocking pool to avoid stalling the async runtime. From 7bcf5d043a1f8d3ec429ea24aa3ee429816dcebc Mon Sep 17 00:00:00 2001 From: Petre_Kharazashvili <96245352+Petre-Kharazashvili@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:42:51 +0400 Subject: [PATCH 085/148] fix(rpc): stop cancelled payload body hash requests (#26569) Co-authored-by: Claude Fable 5 --- crates/rpc/rpc-engine-api/src/engine_api.rs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index 126ccb25c1d..ab08df5e5ab 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -786,6 +786,10 @@ where self.inner.task_spawner.spawn_blocking_task(async move { let mut result = Vec::with_capacity(hashes.len()); for hash in hashes { + if tx.is_closed() { + return; + } + let block_result = inner.provider.block(BlockHashOrNumber::Hash(hash)); match block_result { Ok(block) => { @@ -816,6 +820,10 @@ where let bal_store = self.inner.provider.bal_store().clone(); self.inner.task_spawner.spawn_blocking_task(async move { + if tx.is_closed() { + return; + } + tx.send( bal_store .get_by_hashes(&hashes) From 1eb0c9d21a99bace223437941b7194beac6c011c Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 4 Aug 2026 13:38:37 +0200 Subject: [PATCH 086/148] fix(rpc): accept BlockId for execution witness (#26572) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-api/src/debug.rs | 4 ++-- crates/rpc/rpc-builder/tests/it/http.rs | 31 +++++++++++++++++++++++++ crates/rpc/rpc/src/debug.rs | 8 +++---- docs/vocs/docs/pages/jsonrpc/debug.mdx | 6 +++-- 4 files changed, 41 insertions(+), 8 deletions(-) diff --git a/crates/rpc/rpc-api/src/debug.rs b/crates/rpc/rpc-api/src/debug.rs index e8cdd292fa6..964621d7d2d 100644 --- a/crates/rpc/rpc-api/src/debug.rs +++ b/crates/rpc/rpc-api/src/debug.rs @@ -147,12 +147,12 @@ pub trait DebugApi { /// to their preimages that were required during the execution of the block, including during /// state root recomputation. /// - /// The first argument is the block number or tag. The optional second argument selects the + /// The first argument is the block identifier. The optional second argument selects the /// witness generation mode and defaults to `legacy`. #[method(name = "executionWitness")] async fn debug_execution_witness( &self, - block: BlockNumberOrTag, + block: BlockId, mode: Option, ) -> RpcResult; diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 7bec44f4fcc..1012b86d895 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -40,6 +40,15 @@ fn is_unimplemented(err: jsonrpsee::core::client::Error) -> bool { } } +fn is_invalid_params(err: &jsonrpsee::core::client::Error) -> bool { + match err { + jsonrpsee::core::client::Error::Call(error_obj) => { + error_obj.code() == ErrorCode::InvalidParams.code() + } + _ => false, + } +} + async fn test_rpc_call_ok(client: &HttpClient, method_name: &str, params: ArrayParams) where R: DeserializeOwned, @@ -486,6 +495,28 @@ where ) .await .unwrap_err(); + + for block_id in [ + BlockId::number(0), + BlockId::latest(), + BlockId::hash(B256::ZERO), + BlockId::hash_canonical(B256::ZERO), + ] { + let err = + DebugApiClient::::debug_execution_witness(client, block_id, None) + .await + .unwrap_err(); + assert!(!is_invalid_params(&err)); + } + + let err = DebugApiClient::::debug_execution_witness_by_block_hash( + client, + B256::ZERO, + Some(Default::default()), + ) + .await + .unwrap_err(); + assert!(!is_invalid_params(&err)); } async fn test_basic_net_calls(client: &C) diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 87fddae129d..cf749cfd20f 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -524,15 +524,15 @@ where /// root recomputation. pub async fn debug_execution_witness( &self, - block_id: BlockNumberOrTag, + block_id: BlockId, mode: Option, ) -> Result { let this = self.clone(); let block = this .eth_api() - .recovered_block(block_id.into()) + .recovered_block(block_id) .await? - .ok_or(EthApiError::HeaderNotFound(block_id.into()))?; + .ok_or(EthApiError::HeaderNotFound(block_id))?; self.debug_execution_witness_for_block(block, mode.unwrap_or_default()).await } @@ -953,7 +953,7 @@ where /// Handler for `debug_executionWitness` async fn debug_execution_witness( &self, - block: BlockNumberOrTag, + block: BlockId, mode: Option, ) -> RpcResult { let _permit = self.acquire_trace_permit().await; diff --git a/docs/vocs/docs/pages/jsonrpc/debug.mdx b/docs/vocs/docs/pages/jsonrpc/debug.mdx index 4d65af843ab..a341f5ae32e 100644 --- a/docs/vocs/docs/pages/jsonrpc/debug.mdx +++ b/docs/vocs/docs/pages/jsonrpc/debug.mdx @@ -127,13 +127,15 @@ This method returns nested lists of traces, where the outer list length is the n Allows for re-execution of a block with the purpose of generating an execution witness. The witness comprises a map of all hashed trie nodes to their preimages that were required during the execution of the block, including during state root recomputation. +The `block` parameter accepts a block number, tag, hash, or EIP-1898 block object. An optional second parameter selects the witness generation mode. + | Client | Method invocation | | ------ | ------------------------------------------------------------ | | RPC | `{"method": "debug_executionWitness", "params": [block]}` | ## `debug_executionWitnessByBlockHash` -Similar to [`debug_executionWitness`](#debug_executionwitness), but accepts a block hash instead of a block number or tag. +Similar to [`debug_executionWitness`](#debug_executionwitness), but accepts a block hash and retains its dedicated method for compatibility. | Client | Method invocation | | ------ | ---------------------------------------------------------------------- | @@ -193,4 +195,4 @@ Returns the contract bytecode associated with the given code hash at the specifi | Client | Method invocation | | ------ | ------------------------------------------------------------------------ | -| RPC | `{"method": "debug_codeByHash", "params": [code_hash, block]}` | +| RPC | `{"method": "debug_codeByHash", "params": [code_hash, block]}` | \ No newline at end of file From 9ea6d16ae3b7c8e0a68228b7f6134b64919ae010 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 4 Aug 2026 14:02:50 +0200 Subject: [PATCH 087/148] fix(engine): defer persistence handoff during payload builds (#26559) --- crates/engine/primitives/src/config.rs | 5 +- crates/engine/tree/src/tree/mod.rs | 248 +++++++++++++++++++++---- crates/engine/tree/src/tree/tests.rs | 227 ++++++++++++++++++++++ crates/node/core/src/args/engine.rs | 6 +- crates/payload/basic/src/lib.rs | 11 +- crates/payload/builder/src/service.rs | 31 ++-- docs/vocs/docs/pages/cli/reth/node.mdx | 2 +- 7 files changed, 472 insertions(+), 58 deletions(-) diff --git a/crates/engine/primitives/src/config.rs b/crates/engine/primitives/src/config.rs index 7bf54a66f5f..0bec4746106 100644 --- a/crates/engine/primitives/src/config.rs +++ b/crates/engine/primitives/src/config.rs @@ -196,9 +196,8 @@ pub struct TreeConfig { share_sparse_trie_with_payload_builder: bool, /// Whether to suppress persistence cycles while building a payload. /// - /// When enabled, persistence is deferred from the moment an FCU with payload attributes - /// arrives until the next FCU without attributes. This avoids persistence I/O competing - /// with block building on latency-sensitive chains. + /// When enabled, persistence is deferred while a payload job is active. This avoids + /// persistence I/O competing with block building on latency-sensitive chains. suppress_persistence_during_build: bool, /// Whether to disable BAL (Block Access List, EIP-7928) based parallel execution. /// When disabled, uses the sequential execution path even when a BAL is available. diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index d88d260540d..a1066a1160f 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -23,7 +23,7 @@ use reth_engine_primitives::{ }; use reth_errors::{ConsensusError, ProviderResult}; use reth_evm::ConfigureEvm; -use reth_payload_builder::{BuildNewPayload, PayloadBuilderHandle}; +use reth_payload_builder::{BuildNewPayload, PayloadBuilderHandle, PayloadBuilderLease}; use reth_payload_primitives::{BuiltPayload, NewPayloadError, PayloadAttributes, PayloadTypes}; use reth_primitives_traits::{ FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, @@ -41,7 +41,15 @@ use reth_tasks::{spawn_os_thread, utils::increase_thread_priority}; use reth_trie::ComputedTrieData; use revm::interpreter::debug_unreachable; use state::TreeState; -use std::{fmt::Debug, ops, sync::Arc, time::Duration}; +use std::{ + fmt::Debug, + ops, + sync::{ + atomic::{AtomicUsize, Ordering}, + Arc, + }, + time::Duration, +}; use crossbeam_channel::{Receiver, Sender}; use tokio::sync::{ @@ -353,9 +361,13 @@ where /// Stored here (not in `ExecutedBlock`) to avoid leaking observability concerns into the block /// type. Entries are removed when blocks are persisted or invalidated. execution_timing_stats: B256Map>, - /// Set when an FCU with payload attributes is received, cleared on the next FCU without. - /// Suppresses persistence cycles during payload building. - building_payload: bool, + /// Tracks payload jobs that may still access in-memory overlay state. + payload_builds: PayloadBuildTracker, + /// Notifies the engine when the final active payload job finishes. + payload_build_finished: Receiver<()>, + /// A durable persistence completion whose destructive in-memory handoff is deferred until + /// payload jobs finish. + pending_persisted_handoff: Option, /// Task runtime for spawning blocking work on named, reusable threads. runtime: reth_tasks::Runtime, } @@ -383,6 +395,8 @@ where .field("engine_kind", &self.engine_kind) .field("evm_config", &self.evm_config) .field("execution_timing_stats", &self.execution_timing_stats.len()) + .field("payload_builds_active", &self.payload_builds.is_active()) + .field("pending_persisted_handoff", &self.pending_persisted_handoff) .field("runtime", &self.runtime) .finish() } @@ -427,6 +441,8 @@ where ) -> Self { let (incoming_tx, incoming) = crossbeam_channel::unbounded(); + let (payload_builds, payload_build_finished) = PayloadBuildTracker::new(); + Self { provider, consensus, @@ -445,7 +461,9 @@ where engine_kind, evm_config, execution_timing_stats: B256Map::default(), - building_payload: false, + payload_builds, + payload_build_finished, + pending_persisted_handoff: None, runtime, } } @@ -622,6 +640,12 @@ where return } } + LoopEvent::PayloadBuildFinished => { + if let Err(err) = self.on_payload_build_finished() { + error!(target: "engine::tree", %err, "Payload build completion handling failed"); + return + } + } LoopEvent::Disconnected => { error!(target: "engine::tree", "Channel disconnected"); return @@ -657,12 +681,21 @@ where } } - /// Blocks until the next event is ready: either an incoming engine message or a persistence - /// completion (if one is in progress). + /// Blocks until the next event that can safely be processed is ready. /// - /// Uses biased selection to prioritize persistence completion to update in-memory state and - /// unblock further writes. + /// A pending persisted handoff only accepts the notification that all active payload builds + /// have finished. This keeps queued engine messages from starting replacement payload jobs + /// before the handoff can reclaim the in-memory overlay. Otherwise, uses biased selection to + /// prioritize persistence completion to update in-memory state and unblock further writes. fn wait_for_event(&mut self) -> LoopEvent { + if self.pending_persisted_handoff.is_some() { + self.metrics.engine.backpressure_active.set(0.0); + return match self.payload_build_finished.recv() { + Ok(()) => LoopEvent::PayloadBuildFinished, + Err(_) => LoopEvent::Disconnected, + }; + } + // Take ownership of persistence rx if present let maybe_persistence = self.persistence_state.rx.take(); @@ -680,6 +713,14 @@ where Err(_) => LoopEvent::Disconnected, } }, + recv(self.payload_build_finished) -> result => { + // Put the persistence rx back - we didn't consume it. + self.persistence_state.rx = Some((persistence_rx, start_time, action)); + match result { + Ok(()) => LoopEvent::PayloadBuildFinished, + Err(_) => LoopEvent::Disconnected, + } + }, recv(self.incoming) -> msg => { // Put the persistence rx back - we didn't consume it self.persistence_state.rx = Some((persistence_rx, start_time, action)); @@ -690,10 +731,16 @@ where }, } } else { - // No persistence in progress - just wait on incoming - match self.incoming.recv() { - Ok(m) => LoopEvent::EngineMessage(m), - Err(_) => LoopEvent::Disconnected, + // No persistence in progress - wait on an incoming message or payload job completion. + crossbeam_channel::select_biased! { + recv(self.payload_build_finished) -> result => match result { + Ok(()) => LoopEvent::PayloadBuildFinished, + Err(_) => LoopEvent::Disconnected, + }, + recv(self.incoming) -> msg => match msg { + Ok(m) => LoopEvent::EngineMessage(m), + Err(_) => LoopEvent::Disconnected, + }, } } } @@ -1173,8 +1220,6 @@ where ) -> ProviderResult> { trace!(target: "engine::tree", ?attrs, "invoked forkchoice update"); - self.building_payload = attrs.is_some() && self.config.suppress_persistence_during_build(); - // Record metrics self.record_forkchoice_metrics(); @@ -1431,6 +1476,12 @@ where /// This checks if we need to remove blocks (disk reorg) or save new blocks to disk. /// Persistence completion is handled separately via the `wait_for_event` method. fn advance_persistence(&mut self) -> Result<(), AdvancePersistenceError> { + // A second persistence completion would overwrite the frontier needed by the deferred + // handoff. Wait until the payload jobs using the old overlay have finished. + if self.pending_persisted_handoff.is_some() { + return Ok(()) + } + if !self.persistence_state.in_progress() { if let Some(new_tip_num) = self.find_disk_reorg()? { self.remove_blocks(new_tip_num) @@ -1463,7 +1514,7 @@ where if let Some((rx, start_time, action)) = self.persistence_state.rx.take() { debug!(target: "engine::tree", ?action, "waiting for in-flight persistence"); let result = rx.recv().map_err(|_| AdvancePersistenceError::ChannelClosed)?; - self.on_persistence_complete(result, start_time)?; + self.finish_persistence(result, start_time); continue } @@ -1515,25 +1566,67 @@ where result: PersistenceResult, start_time: Instant, ) -> Result<(), AdvancePersistenceError> { + let handoff = self.finish_persistence(result, start_time); + + if self.payload_builds.is_active() { + debug_assert!( + self.pending_persisted_handoff.is_none(), + "a new persistence task must not start while its predecessor handoff is pending" + ); + debug!(target: "engine::tree", "Deferring persisted in-memory handoff until payload jobs finish"); + self.pending_persisted_handoff = Some(handoff); + return Ok(()) + } + + self.on_persisted_handoff(handoff) + } + + /// Records a completed persistence operation and returns its deferred in-memory handoff. + fn finish_persistence( + &mut self, + result: PersistenceResult, + start_time: Instant, + ) -> PersistenceResult { self.metrics.engine.persistence_duration.record(start_time.elapsed()); - let commit_duration = result.commit_duration; - let last_persisted_block = result.last_block; - let last_persisted_block_number = last_persisted_block.number; - let last_state_trie_persisted_block = result.last_state_trie_block; + let last_block = result.last_block; + let last_state_trie_block = result.last_state_trie_block; debug_assert!( - last_state_trie_persisted_block.number <= last_persisted_block.number, + last_state_trie_block.number <= last_block.number, "state/trie frontier cannot exceed the last persisted block" ); - debug!(target: "engine::tree", ?last_persisted_block, ?last_state_trie_persisted_block, elapsed=?start_time.elapsed(), "Finished persisting, calling finish"); - self.persistence_state.finish(last_persisted_block, last_state_trie_persisted_block); + debug!(target: "engine::tree", ?last_block, ?last_state_trie_block, elapsed=?start_time.elapsed(), "Finished persisting, calling finish"); + self.persistence_state.finish(last_block, last_state_trie_block); + + result + } + + /// Applies the destructive in-memory portion of a completed persistence operation. + fn on_persisted_handoff( + &mut self, + handoff: PersistenceResult, + ) -> Result<(), AdvancePersistenceError> { + if handoff.last_block != self.persistence_state.last_persisted_block || + handoff.last_state_trie_block != + self.persistence_state.last_state_trie_persisted_block + { + debug!( + target: "engine::tree", + handoff_last_block = ?handoff.last_block, + current_last_block = ?self.persistence_state.last_persisted_block, + "Discarding stale persisted handoff" + ); + return Ok(()) + } + + let PersistenceResult { last_block, commit_duration, .. } = handoff; + let last_block_number = last_block.number; // Evict cached changesets for blocks below the eviction threshold. // Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect // the finalized block if set. - let min_threshold = - last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS); + let min_threshold = last_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS); let eviction_threshold = if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() { // Use the minimum of finalized block and retention threshold to be conservative @@ -1544,7 +1637,7 @@ where }; debug!( target: "engine::tree", - last_persisted = last_persisted_block_number, + last_persisted = last_block_number, finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number), eviction_threshold, "Evicting changesets below threshold" @@ -1553,7 +1646,20 @@ where self.on_new_persisted_block()?; - self.purge_timing_stats(last_persisted_block_number, commit_duration); + self.purge_timing_stats(last_block_number, commit_duration); + + Ok(()) + } + + /// Releases work deferred until all payload jobs have finished. + fn on_payload_build_finished(&mut self) -> Result<(), AdvancePersistenceError> { + if self.payload_builds.is_active() { + return Ok(()) + } + + if let Some(handoff) = self.pending_persisted_handoff.take() { + self.on_persisted_handoff(handoff)?; + } Ok(()) } @@ -2081,10 +2187,13 @@ where "backfill action should only be emitted when backfill is idle" ); - if self.persistence_state.in_progress() { - // backfill sync and persisting data are mutually exclusive, so we can't start - // backfill while we're still persisting - debug!(target: "engine::tree", "skipping backfill file while persistence task is active"); + if self.payload_builds.is_active() || + self.persistence_state.in_progress() || + self.pending_persisted_handoff.is_some() + { + // Backfill can remove the same in-memory blocks as an active payload job or a + // persisted handoff, so it must not start until the next sync trigger. + debug!(target: "engine::tree", "skipping backfill while in-memory overlay is in use"); return } @@ -2115,7 +2224,10 @@ where let (new_db_tip, new_partial_state_trie) = match target { PersistTarget::Head => (canonical_head_number, canonical_head_number), PersistTarget::Threshold => { - if self.building_payload || !self.backfill_sync_state.is_idle() { + if (self.config.suppress_persistence_during_build() && + self.payload_builds.is_active()) || + !self.backfill_sync_state.is_idle() + { return None } @@ -3341,12 +3453,19 @@ where // payloadAttributes is not null and the forkchoice state has been updated successfully. // The build process is specified in the Payload building section. - let resources = self.payload_validator.payload_builder_resources( - state.head_block_hash, - head, - attributes.timestamp(), - &mut self.state, - ); + // Acquire this before preparing resources because state-root setup can already start + // workers that need the current in-memory overlay. + let payload_build = self.payload_builds.acquire(); + + let resources = self + .payload_validator + .payload_builder_resources( + state.head_block_hash, + head, + attributes.timestamp(), + &mut self.state, + ) + .with_lease(PayloadBuilderLease::new(payload_build)); // send the payload to the builder and return the receiver for the pending payload // id, initiating payload job is handled asynchronously @@ -3450,10 +3569,61 @@ where /// When the persistence operation started. start_time: Instant, }, + /// The last active payload job has finished, so deferred in-memory work may proceed. + PayloadBuildFinished, /// A channel was disconnected. Disconnected, } +/// Tracks payload jobs that may access the current in-memory overlay. +#[derive(Clone, Debug)] +struct PayloadBuildTracker { + active: Arc, + finished_tx: Sender<()>, +} + +impl PayloadBuildTracker { + /// Creates a tracker and a receiver notified when its active count reaches zero. + fn new() -> (Self, Receiver<()>) { + let (finished_tx, finished_rx) = crossbeam_channel::bounded(1); + (Self { active: Arc::new(AtomicUsize::new(0)), finished_tx }, finished_rx) + } + + /// Acquires a lease for one payload job. + fn acquire(&self) -> PayloadBuildLease { + self.active.fetch_add(1, Ordering::AcqRel); + PayloadBuildLease { + active: Arc::clone(&self.active), + finished_tx: self.finished_tx.clone(), + } + } + + /// Returns whether at least one payload job is active. + fn is_active(&self) -> bool { + self.active.load(Ordering::Acquire) != 0 + } +} + +/// A lease held for the lifetime of a payload job. +#[derive(Debug)] +struct PayloadBuildLease { + active: Arc, + finished_tx: Sender<()>, +} + +impl Drop for PayloadBuildLease { + fn drop(&mut self) { + let previous = self.active.fetch_sub(1, Ordering::AcqRel); + debug_assert!(previous > 0, "payload build lease count underflow"); + + if previous == 1 { + // The bounded channel coalesces completion notifications. The engine always checks + // the counter again before applying a pending handoff. + let _ = self.finished_tx.try_send(()); + } + } +} + /// Block inclusion can be valid, accepted, or invalid. Invalid blocks are returned as an error /// variant. /// diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index cec99a52726..6ded5d35e5c 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -642,6 +642,229 @@ fn on_new_persisted_block_skips_sparse_trie_prune_when_state_root_task_disabled( } } +#[test] +fn payload_build_tracker_notifies_after_last_lease_drops() { + let (tracker, finished) = PayloadBuildTracker::new(); + let first = tracker.acquire(); + let second = tracker.acquire(); + + assert!(tracker.is_active()); + drop(first); + assert!(tracker.is_active()); + assert!(finished.try_recv().is_err()); + + drop(second); + assert!(!tracker.is_active()); + assert_eq!(finished.try_recv(), Ok(())); +} + +#[test] +fn persistence_handoff_waits_for_active_payload_jobs() { + let config = TreeConfig::default() + .with_has_enough_parallelism(true) + .with_persistence_threshold(0) + .with_memory_block_buffer_target(0); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); + let mut test_harness = + TestHarness::with_config(MAINNET.clone(), config).with_blocks(blocks.clone()); + let persisted = blocks[0].recovered_block().num_hash(); + let persisted_hash = persisted.hash; + let first = test_harness.tree.payload_builds.acquire(); + let second = test_harness.tree.payload_builds.acquire(); + + test_harness + .tree + .on_persistence_complete( + PersistenceResult { + last_block: persisted, + last_state_trie_block: persisted, + commit_duration: Some(Duration::ZERO), + }, + Instant::now(), + ) + .unwrap(); + + // Durability advances immediately, but the in-memory overlay remains available to jobs. + assert_eq!(test_harness.tree.persistence_state.last_persisted_block, persisted); + assert!(test_harness.tree.pending_persisted_handoff.is_some()); + assert!(test_harness.tree.state.tree_state.contains_hash(&persisted_hash)); + assert!(test_harness.tree.canonical_in_memory_state.state_by_hash(persisted_hash).is_some()); + + // Do not let another persistence action overwrite the deferred handoff's frontier. + test_harness.tree.advance_persistence().unwrap(); + assert!(!test_harness.tree.persistence_state.in_progress()); + + drop(first); + assert!(test_harness.tree.payload_build_finished.try_recv().is_err()); + assert!(test_harness.tree.pending_persisted_handoff.is_some()); + + drop(second); + assert!(matches!(test_harness.tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished)); + test_harness.tree.on_payload_build_finished().unwrap(); + + assert!(test_harness.tree.pending_persisted_handoff.is_none()); + assert!(!test_harness.tree.state.tree_state.contains_hash(&persisted_hash)); + assert!(test_harness.tree.canonical_in_memory_state.state_by_hash(persisted_hash).is_none()); +} + +#[test] +fn pending_persisted_handoff_blocks_engine_messages_until_payload_build_finishes() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); + let mut test_harness = TestHarness::with_config( + MAINNET.clone(), + TreeConfig::default().with_has_enough_parallelism(true), + ) + .with_blocks(blocks.clone()); + let persisted = blocks[0].recovered_block().num_hash(); + let payload_build = test_harness.tree.payload_builds.acquire(); + + test_harness + .tree + .on_persistence_complete( + PersistenceResult { + last_block: persisted, + last_state_trie_block: persisted, + commit_duration: Some(Duration::ZERO), + }, + Instant::now(), + ) + .unwrap(); + assert!(test_harness.tree.pending_persisted_handoff.is_some()); + + // Forkchoice updates received while the handoff is pending could start replacement payload + // builds and keep the final completion notification from reaching the tree. Queue a generic + // engine message to prove the gate leaves every message upstream until then. + test_harness.to_tree_tx.send(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)).unwrap(); + + let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0); + let (event_tx, event_rx) = std::sync::mpsc::sync_channel(1); + std::thread::scope(|scope| { + let tree = &mut test_harness.tree; + scope.spawn(move || { + started_tx.send(()).unwrap(); + event_tx + .send(matches!(tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished)) + .unwrap(); + }); + + started_rx.recv().unwrap(); + assert!( + event_rx.recv_timeout(Duration::from_millis(100)).is_err(), + "a pending handoff must prevent queued engine messages from being processed" + ); + + drop(payload_build); + assert_eq!(event_rx.recv_timeout(Duration::from_secs(1)), Ok(true)); + }); + + test_harness.tree.on_payload_build_finished().unwrap(); + assert!(test_harness.tree.pending_persisted_handoff.is_none()); + assert_matches!( + test_harness.tree.wait_for_event(), + super::LoopEvent::EngineMessage(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)) + ); +} + +#[test] +fn persist_until_complete_updates_frontiers_without_in_memory_handoff() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); + let mut test_harness = TestHarness::new(MAINNET.clone()).with_blocks(blocks.clone()); + let persisted = blocks.last().unwrap().recovered_block().num_hash(); + let persisted_hash = persisted.hash; + let (tx, rx) = crossbeam_channel::bounded(1); + + tx.send(PersistenceResult { + last_block: persisted, + last_state_trie_block: persisted, + commit_duration: Some(Duration::ZERO), + }) + .unwrap(); + test_harness.tree.persistence_state.start_save(persisted, rx); + + test_harness.tree.persist_until_complete().unwrap(); + + // Shutdown waits for the durable write, but process teardown does not need the destructive + // in-memory handoff that normally follows a completed persistence task. + assert_eq!(test_harness.tree.persistence_state.last_persisted_block, persisted); + assert_eq!(test_harness.tree.persistence_state.last_state_trie_persisted_block, persisted); + assert!(!test_harness.tree.persistence_state.in_progress()); + assert!(test_harness.tree.pending_persisted_handoff.is_none()); + assert!(test_harness.tree.state.tree_state.contains_hash(&persisted_hash)); + assert!(test_harness.tree.canonical_in_memory_state.state_by_hash(persisted_hash).is_some()); +} + +#[test] +fn backfill_action_skips_while_payload_build_is_active() { + let mut test_harness = TestHarness::new(MAINNET.clone()); + let payload_build = test_harness.tree.payload_builds.acquire(); + let action = BackfillAction::Start(B256::random().into()); + + test_harness.tree.emit_event(EngineApiEvent::BackfillAction(action)); + assert!(test_harness.tree.pending_persisted_handoff.is_none()); + assert!(test_harness.tree.backfill_sync_state.is_idle()); + assert!(test_harness.from_tree_rx.try_recv().is_err()); + + drop(payload_build); + assert!(matches!(test_harness.tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished)); + test_harness.tree.on_payload_build_finished().unwrap(); + + // The skipped action is not queued for a later replay. + assert!(test_harness.tree.backfill_sync_state.is_idle()); + assert!(test_harness.from_tree_rx.try_recv().is_err()); +} + +#[test] +fn backfill_action_skips_while_persisted_handoff_is_pending() { + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); + let mut test_harness = TestHarness::with_config( + MAINNET.clone(), + TreeConfig::default().with_has_enough_parallelism(true), + ) + .with_blocks(blocks.clone()); + let persisted = blocks[0].recovered_block().num_hash(); + let payload_build = test_harness.tree.payload_builds.acquire(); + + test_harness + .tree + .on_persistence_complete( + PersistenceResult { + last_block: persisted, + last_state_trie_block: persisted, + commit_duration: Some(Duration::ZERO), + }, + Instant::now(), + ) + .unwrap(); + + drop(payload_build); + assert!(!test_harness.tree.payload_builds.is_active()); + assert!(test_harness.tree.pending_persisted_handoff.is_some()); + + test_harness + .tree + .emit_event(EngineApiEvent::BackfillAction(BackfillAction::Start(B256::random().into()))); + + assert!(test_harness.tree.backfill_sync_state.is_idle()); + assert!(test_harness.from_tree_rx.try_recv().is_err()); +} + +#[test] +fn configured_persistence_suppression_tracks_payload_job_lifetime() { + let config = TreeConfig::default() + .with_has_enough_parallelism(true) + .with_persistence_threshold(0) + .with_memory_block_buffer_target(0) + .with_suppress_persistence_during_build(true); + let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); + let test_harness = TestHarness::with_config(MAINNET.clone(), config).with_blocks(blocks); + let payload_build = test_harness.tree.payload_builds.acquire(); + + assert!(test_harness.tree.get_save_blocks_input(PersistTarget::Threshold).is_none()); + + drop(payload_build); + assert!(test_harness.tree.get_save_blocks_input(PersistTarget::Threshold).is_some()); +} + #[test] fn remove_blocks_clears_pending_sparse_trie_prune_request() { let mut test_harness = TestHarness::new(MAINNET.clone()); @@ -690,6 +913,10 @@ fn process_payload_attributes_shares_sparse_trie_during_validation_fallback() { panic!("expected build new payload command") }; assert!(input.resources.state_root_handle().is_some()); + assert!(test_harness.tree.payload_builds.is_active()); + + drop(input); + assert!(!test_harness.tree.payload_builds.is_active()); } #[tokio::test] diff --git a/crates/node/core/src/args/engine.rs b/crates/node/core/src/args/engine.rs index bfd54c40669..06df7745ece 100644 --- a/crates/node/core/src/args/engine.rs +++ b/crates/node/core/src/args/engine.rs @@ -543,9 +543,9 @@ pub struct EngineArgs { /// Suppress persistence while building a payload. /// - /// When enabled, persistence cycles are deferred from the moment an FCU with payload - /// attributes arrives until the next FCU clears the build. Useful on chains with short - /// block times where persistence I/O can interfere with block building latency. + /// When enabled, persistence cycles are deferred while a payload build is active. Useful on + /// chains with short block times where persistence I/O can interfere with block building + /// latency. #[arg( long = "engine.suppress-persistence-during-build", default_value_t = DefaultEngineValues::get_global().suppress_persistence_during_build, diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index a78f7967668..f9c18dfc00d 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -16,7 +16,8 @@ use futures_util::FutureExt; use reth_chain_state::CanonStateNotification; use reth_execution_cache::SavedCache; use reth_payload_builder::{ - BuildNewPayload, KeepPayloadJobAlive, PayloadId, PayloadJob, PayloadJobGenerator, + BuildNewPayload, KeepPayloadJobAlive, PayloadBuilderLease, PayloadId, PayloadJob, + PayloadJobGenerator, }; use reth_payload_builder_primitives::PayloadBuilderError; use reth_payload_primitives::{BuiltPayload, PayloadAttributes, PayloadKind}; @@ -200,6 +201,7 @@ where cached_reads, execution_cache: resources.take_execution_cache(), state_root_handle: resources.take_state_root_handle(), + leases: resources.take_leases(), payload_task_guard: self.payload_task_guard.clone(), metrics: Default::default(), builder: self.builder.clone(), @@ -390,6 +392,11 @@ where execution_cache: Option, /// Optional state-root task handle, shared with the engine. state_root_handle: Option, + /// Lifecycle leases shared with the payload-builder service. + /// + /// Every detached build task clones these so that the loaned resources remain available until + /// `try_build` completes, even if the payload job is resolved first. + leases: Vec, /// metrics for this type metrics: PayloadBuilderMetrics, /// The type responsible for building payloads. @@ -417,6 +424,7 @@ where let cached_reads = self.cached_reads.take().unwrap_or_default(); let execution_cache = self.execution_cache.clone(); let state_root_handle = self.state_root_handle.take(); + let leases = self.leases.clone(); let builder = self.builder.clone(); let executor = self.executor.clone(); self.executor.spawn_task(async move { @@ -433,6 +441,7 @@ where best_payload, }; let result = builder.try_build(args); + drop(leases); let _ = tx.send(result); }); }); diff --git a/crates/payload/builder/src/service.rs b/crates/payload/builder/src/service.rs index fc7c684ad45..4d6e4a62f70 100644 --- a/crates/payload/builder/src/service.rs +++ b/crates/payload/builder/src/service.rs @@ -445,7 +445,7 @@ where // drain all requests while let Poll::Ready(Some(cmd)) = this.command_rx.poll_next_unpin(cx) { match cmd { - PayloadServiceCommand::BuildNewPayload(mut input, job_span, tx) => { + PayloadServiceCommand::BuildNewPayload(input, job_span, tx) => { let id = input.payload_id(); let mut res = Ok(id); let parent = input.parent_hash; @@ -455,7 +455,10 @@ where } else { let start = Instant::now(); let attributes = input.attributes.clone(); - let leases = input.resources.take_leases(); + // Keep a service-owned reference for generic payload jobs. Builders + // that move work into detached tasks can retain their own references + // from `resources` until those tasks exit. + let leases = input.resources.clone_leases(); let job_result = { let _entered = job_span.enter(); this.generator.new_payload_job(*input, id) @@ -581,7 +584,7 @@ pub struct PayloadBuilderResources { execution_cache: Option, /// Optional handle to a background state-root task. state_root_handle: Option, - /// Lifecycle leases retained by the service while the payload job is active. + /// Lifecycle leases retained by the service or by detached payload build tasks. leases: Vec, } @@ -594,7 +597,7 @@ impl PayloadBuilderResources { Self { execution_cache, state_root_handle, leases: Vec::new() } } - /// Adds a lease that remains active for the lifetime of the payload job. + /// Adds a lease for this payload build. pub fn with_lease(mut self, lease: PayloadBuilderLease) -> Self { self.leases.push(lease); self @@ -620,21 +623,27 @@ impl PayloadBuilderResources { self.state_root_handle.take() } - /// Takes the lifecycle leases that the service must retain for this job. - fn take_leases(&mut self) -> Vec { + /// Takes lifecycle leases for a payload job that owns detached work. + pub fn take_leases(&mut self) -> Vec { std::mem::take(&mut self.leases) } + + /// Clones lifecycle leases for the payload builder service to retain. + fn clone_leases(&self) -> Vec { + self.leases.clone() + } } -/// Keeps a loaned resource active for the lifetime of a payload job. +/// Keeps a loaned resource active until the last lease clone is dropped. +#[derive(Clone)] pub struct PayloadBuilderLease { - _lease: Box, + _lease: Arc, } impl PayloadBuilderLease { - /// Wraps a lease that releases its resource when dropped. - pub fn new(lease: impl Send + 'static) -> Self { - Self { _lease: Box::new(lease) } + /// Wraps a lease that releases its resource when the last clone is dropped. + pub fn new(lease: impl Send + Sync + 'static) -> Self { + Self { _lease: Arc::new(lease) } } } diff --git a/docs/vocs/docs/pages/cli/reth/node.mdx b/docs/vocs/docs/pages/cli/reth/node.mdx index fe6645912bb..d3835ed45d4 100644 --- a/docs/vocs/docs/pages/cli/reth/node.mdx +++ b/docs/vocs/docs/pages/cli/reth/node.mdx @@ -1111,7 +1111,7 @@ Engine: --engine.suppress-persistence-during-build Suppress persistence while building a payload. - When enabled, persistence cycles are deferred from the moment an FCU with payload attributes arrives until the next FCU clears the build. Useful on chains with short block times where persistence I/O can interfere with block building latency. + When enabled, persistence cycles are deferred while a payload build is active. Useful on chains with short block times where persistence I/O can interfere with block building latency. --engine.disable-bal-parallel-execution Disable BAL (Block Access List, EIP-7928) based parallel execution From 0b0f7dbadee2ffc158fae22797bf443af564af07 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Tue, 4 Aug 2026 14:42:44 +0200 Subject: [PATCH 088/148] feat(stages): handle partial trie unwinds (#26543) --- crates/node/builder/src/launch/common.rs | 363 +++++++++++++++--- crates/stages/stages/src/stages/merkle.rs | 30 +- crates/storage/db-api/src/models/metadata.rs | 9 + .../src/providers/database/provider.rs | 23 +- crates/storage/storage-api/src/metadata.rs | 5 + crates/trie/trie/src/trie.rs | 34 ++ crates/trie/trie/src/walker.rs | 109 ++++++ 7 files changed, 516 insertions(+), 57 deletions(-) diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index cdf3f978543..498c65b3fab 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -41,7 +41,9 @@ use rayon::ThreadPoolBuilder; use reth_chainspec::{Chain, EthChainSpec, EthereumHardforks}; use reth_config::{config::EtlConfig, PruneConfig}; use reth_consensus::noop::NoopConsensus; -use reth_db_api::{database::Database, database_metrics::DatabaseMetrics}; +use reth_db_api::{ + database::Database, database_metrics::DatabaseMetrics, models::PartialStateTrieUnwindMarker, +}; use reth_db_common::init::{ init_genesis_with_settings, init_genesis_with_settings_and_validate, InitStorageError, }; @@ -69,7 +71,8 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, - BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, InMemoryBalStore, ProviderError, + BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, + DatabaseProviderFactory, InMemoryBalStore, MetadataProvider, MetadataWriter, ProviderError, ProviderFactory, ProviderResult, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder, StaticFileProviderFactory, StorageSettingsCache, }; @@ -77,8 +80,9 @@ use reth_prune::{PruneMode, PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; use reth_rpc_layer::JwtSecret; use reth_stages::{ - sets::DefaultStages, stages::EraImportSource, MetricEvent, PipelineBuilder, PipelineTarget, - StageId, StageSet, + sets::DefaultStages, + stages::{EraImportSource, MerkleStage}, + MetricEvent, PipelineBuilder, PipelineTarget, StageId, StageSet, }; use reth_static_file::{blocks_per_file_for_prune_distance, StaticFileProducer, StaticFileSegment}; use reth_storage_overlay::OverlayManager; @@ -544,64 +548,122 @@ where // the unwind targets for each storage layer if inconsistencies are // found. let (rocksdb_unwind, static_file_unwind) = factory.check_consistency()?; + let provider_ro = factory.database_provider_ro()?; + // Finish is committed before Merkle during unwind, so this marker is authoritative when + // resuming an interrupted partial trie unwind. + let (partial_trie_unwind, has_persisted_partial_trie_unwind) = + get_partial_trie_unwind_marker(&provider_ro)?; + drop(provider_ro); + let persist_partial_trie_unwind = + !has_persisted_partial_trie_unwind && partial_trie_unwind.is_some(); + let partial_trie_unwind_target = + partial_trie_unwind.map(|marker| marker.partial_state_trie); + // Recover the partial state trie first. Its unwind enables + // `walk_all_changed_branch_children`, which is more expensive than a normal unwind, so + // it only runs to the partial trie target. A lower storage-layer target is then unwound + // normally. + let storage_unwind = [rocksdb_unwind, static_file_unwind].into_iter().flatten().min(); + let storage_unwind = storage_unwind.filter(|unwind_block| { + partial_trie_unwind_target.is_none_or(|partial_trie| *unwind_block < partial_trie) + }); - // Take the minimum block number to ensure all storage layers are consistent. - let unwind_target = [rocksdb_unwind, static_file_unwind].into_iter().flatten().min(); - - if let Some(unwind_block) = unwind_target { - // Highly unlikely to happen, and given its destructive nature, it's better to panic - // instead. Unwinding to 0 would leave MDBX with a huge free list size. - let inconsistency_source = match (rocksdb_unwind, static_file_unwind) { - (Some(_), Some(_)) => "RocksDB and static file", - (Some(_), None) => "RocksDB", - (None, Some(_)) => "static file", - (None, None) => unreachable!(), - }; - assert_ne!( - unwind_block, 0, - "A {} inconsistency was found that would trigger an unwind to block 0", - inconsistency_source - ); - - let unwind_target = PipelineTarget::Unwind(unwind_block); - - info!(target: "reth::cli", %unwind_target, %inconsistency_source, "Executing unwind after consistency check."); - - let (_tip_tx, tip_rx) = watch::channel(B256::ZERO); - - // Builds an unwind-only pipeline - let pipeline = PipelineBuilder::default() - .add_stages( - DefaultStages::new( - factory.clone(), - tip_rx, - Arc::new(NoopConsensus::default()), - NoopHeaderDownloader::default(), - NoopBodiesDownloader::default(), - NoopEvmConfig::::default(), - self.toml_config().stages.clone(), - self.prune_modes(), - None, - ) - .builder() - .disable_all(disabled_stages), + if partial_trie_unwind_target.is_some() || storage_unwind.is_some() { + let build_unwind_pipeline = |walk_all_changed_branch_children| { + let (_tip_tx, tip_rx) = watch::channel(B256::ZERO); + let mut stages = DefaultStages::new( + factory.clone(), + tip_rx, + Arc::new(NoopConsensus::default()), + NoopHeaderDownloader::default(), + NoopBodiesDownloader::default(), + NoopEvmConfig::::default(), + self.toml_config().stages.clone(), + self.prune_modes(), + None, ) - .build( + .builder() + .disable_all(disabled_stages); + + if walk_all_changed_branch_children { + // Partial trie recovery is not complete until Merkle has unwound. + stages = + stages.set(MerkleStage::new_unwind(true)).enable(StageId::MerkleUnwind); + } + + PipelineBuilder::default().add_stages(stages).build( factory.clone(), StaticFileProducer::new(factory.clone(), self.prune_modes()), + ) + }; + let mut unwinds = Vec::with_capacity(2); + + if let Some(unwind_block) = partial_trie_unwind_target { + unwinds.push(( + PipelineTarget::Unwind(unwind_block), + "partial state trie".to_owned(), + build_unwind_pipeline(true), + true, + )); + } + + if let Some(unwind_block) = storage_unwind { + // Highly unlikely to happen, and given its destructive nature, it's better to + // panic instead. Unwinding to 0 would leave MDBX with a huge free list size. + let inconsistency_source = match (rocksdb_unwind, static_file_unwind) { + (Some(_), Some(_)) => "RocksDB and static file", + (Some(_), None) => "RocksDB", + (None, Some(_)) => "static file", + (None, None) => unreachable!(), + }; + assert_ne!( + unwind_block, 0, + "A {inconsistency_source} inconsistency was found that would trigger an unwind to block 0" ); + unwinds.push(( + PipelineTarget::Unwind(unwind_block), + inconsistency_source.to_owned(), + build_unwind_pipeline(false), + false, + )); + } + + if persist_partial_trie_unwind { + // The marker must be durable before any unwind stage can commit. + let provider_rw = factory.database_provider_rw()?; + write_partial_trie_unwind_marker( + &provider_rw, + partial_trie_unwind.expect("partial trie unwind marker must exist"), + )?; + provider_rw.commit()?; + } - // Unwinds to block let (tx, rx) = oneshot::channel(); + let factory = factory.clone(); // Pipeline should be run as blocking and panic if it fails. self.task_executor().spawn_critical_blocking_task("pipeline task", async move { - let (_, result) = pipeline.run_as_fut(Some(unwind_target)).await; + let result: Result<(), reth_stages::PipelineError> = async { + for (unwind_target, inconsistency_source, pipeline, clear_partial_trie_unwind) in + unwinds + { + info!(target: "reth::cli", %unwind_target, %inconsistency_source, "Executing unwind after consistency check."); + let (_, result) = pipeline.run_as_fut(Some(unwind_target)).await; + result.inspect_err(|err| { + error!(target: "reth::cli", %unwind_target, %inconsistency_source, %err, "failed to run unwind"); + })?; + + if clear_partial_trie_unwind { + let provider_rw = factory.database_provider_rw()?; + delete_partial_trie_unwind_marker(&provider_rw)?; + provider_rw.commit()?; + } + } + Ok(()) + } + .await; let _ = tx.send(result); }); - rx.await?.inspect_err(|err| { - error!(target: "reth::cli", %unwind_target, %inconsistency_source, %err, "failed to run unwind") - })?; + rx.await??; } Ok(factory) @@ -1336,14 +1398,99 @@ pub fn metrics_hooks(provider_factory: &ProviderFactory) .build() } +fn get_partial_trie_unwind_marker( + provider: &(impl MetadataProvider + StageCheckpointReader), +) -> ProviderResult<(Option, bool)> { + if let Some(marker) = provider.get_metadata(PARTIAL_STATE_TRIE_UNWIND_METADATA_KEY)? { + let marker = serde_json::from_slice::(&marker) + .map_err(ProviderError::other)?; + if marker.partial_state_trie >= marker.finish_block_number { + return Err(ProviderError::other(std::io::Error::other(format!( + "partial state trie unwind target #{} is not below original Finish #{}", + marker.partial_state_trie, marker.finish_block_number, + )))) + } + return Ok((Some(marker), true)) + } + + let Some(finish_checkpoint) = provider.get_stage_checkpoint(StageId::Finish)? else { + return Ok((None, false)) + }; + let Some(partial_state_trie) = + finish_checkpoint.finish_stage_checkpoint().and_then(|finish| finish.partial_state_trie()) + else { + return Ok((None, false)) + }; + + if partial_state_trie > finish_checkpoint.block_number { + return Err(ProviderError::other(std::io::Error::other(format!( + "partial state trie frontier #{partial_state_trie} is ahead of Finish #{}", + finish_checkpoint.block_number, + )))) + } + + Ok(( + (partial_state_trie < finish_checkpoint.block_number).then_some( + PartialStateTrieUnwindMarker { + finish_block_number: finish_checkpoint.block_number, + partial_state_trie, + }, + ), + false, + )) +} + +/// Metadata key for a partial state trie unwind that has not completed yet. +const PARTIAL_STATE_TRIE_UNWIND_METADATA_KEY: &str = "partial_state_trie_unwind"; + +fn write_partial_trie_unwind_marker( + provider: &impl MetadataWriter, + marker: PartialStateTrieUnwindMarker, +) -> ProviderResult<()> { + provider.write_metadata( + PARTIAL_STATE_TRIE_UNWIND_METADATA_KEY, + serde_json::to_vec(&marker).map_err(ProviderError::other)?, + ) +} + +fn delete_partial_trie_unwind_marker(provider: &impl MetadataWriter) -> ProviderResult<()> { + provider.delete_metadata(PARTIAL_STATE_TRIE_UNWIND_METADATA_KEY) +} + #[cfg(test)] mod tests { - use super::{LaunchContext, NodeConfig}; + use super::{get_partial_trie_unwind_marker, LaunchContext, NodeConfig}; use reth_config::Config; + use reth_db_api::models::PartialStateTrieUnwindMarker; use reth_node_core::args::PruningArgs; + use reth_provider::{MetadataProvider, ProviderResult, StageCheckpointReader}; + use reth_stages::{FinishCheckpoint, StageCheckpoint, StageId}; const EXTENSION: &str = "toml"; + struct MockProvider(Option>, Option); + + impl MetadataProvider for MockProvider { + fn get_metadata(&self, _: &str) -> ProviderResult>> { + Ok(self.0.clone()) + } + } + + impl StageCheckpointReader for MockProvider { + fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult> { + assert_eq!(id, StageId::Finish); + Ok(self.1) + } + + fn get_stage_checkpoint_progress(&self, _: StageId) -> ProviderResult>> { + Ok(None) + } + + fn get_all_checkpoints(&self) -> ProviderResult> { + Ok(Vec::new()) + } + } + fn with_tempdir(filename: &str, proc: fn(&std::path::Path)) { let temp_dir = tempfile::tempdir().unwrap(); let config_path = temp_dir.path().join(filename).with_extension(EXTENSION); @@ -1392,4 +1539,118 @@ mod tests { assert_eq!(reth_config, loaded_config); }) } + + #[test] + fn get_partial_trie_unwind_marker_uses_partial_finish_checkpoint() { + let finish_checkpoint = StageCheckpoint::new(42) + .with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(21) }); + let expected = + finish_checkpoint.finish_stage_checkpoint().unwrap().partial_state_trie().map( + |partial_state_trie| PartialStateTrieUnwindMarker { + finish_block_number: finish_checkpoint.block_number, + partial_state_trie, + }, + ); + + assert_eq!( + get_partial_trie_unwind_marker(&MockProvider(None, Some(finish_checkpoint))).unwrap(), + (expected, false) + ); + + let genesis_checkpoint = StageCheckpoint::new(42) + .with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(0) }); + let expected = + genesis_checkpoint.finish_stage_checkpoint().unwrap().partial_state_trie().map( + |partial_state_trie| PartialStateTrieUnwindMarker { + finish_block_number: genesis_checkpoint.block_number, + partial_state_trie, + }, + ); + + assert_eq!( + get_partial_trie_unwind_marker(&MockProvider(None, Some(genesis_checkpoint))).unwrap(), + (expected, false) + ); + } + + #[test] + fn get_partial_trie_unwind_marker_resumes_persisted_unwind() { + let marker = + PartialStateTrieUnwindMarker { finish_block_number: 42, partial_state_trie: 21 }; + + assert_eq!( + get_partial_trie_unwind_marker(&MockProvider( + Some(serde_json::to_vec(&marker).unwrap()), + Some(StageCheckpoint::new(21)), + ),) + .unwrap(), + (Some(marker), true) + ); + assert_eq!( + get_partial_trie_unwind_marker(&MockProvider( + Some(serde_json::to_vec(&marker).unwrap()), + None + ),) + .unwrap(), + (Some(marker), true) + ); + } + + #[test] + fn get_partial_trie_unwind_marker_ignores_non_lagging_or_missing_partial_checkpoint() { + let matching_finish_checkpoint = StageCheckpoint::new(42) + .with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(42) }); + let ahead_finish_checkpoint = StageCheckpoint::new(42) + .with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: Some(43) }); + let missing_partial_finish_checkpoint = StageCheckpoint::new(42) + .with_finish_stage_checkpoint(FinishCheckpoint { partial_state_trie: None }); + + assert_eq!( + get_partial_trie_unwind_marker(&MockProvider(None, Some(matching_finish_checkpoint)),) + .unwrap(), + (None, false) + ); + assert_eq!( + get_partial_trie_unwind_marker(&MockProvider( + None, + Some(missing_partial_finish_checkpoint) + ),) + .unwrap(), + (None, false) + ); + assert_eq!( + get_partial_trie_unwind_marker(&MockProvider(None, None)).unwrap(), + (None, false) + ); + + let partial_frontier = ahead_finish_checkpoint + .finish_stage_checkpoint() + .and_then(|finish| finish.partial_state_trie()); + let result = + get_partial_trie_unwind_marker(&MockProvider(None, Some(ahead_finish_checkpoint))); + if partial_frontier.is_some() { + let error = result.unwrap_err(); + assert!(error.to_string().contains("ahead of Finish"), "unexpected error: {error}"); + } else { + assert_eq!(result.unwrap(), (None, false)); + } + } + + #[test] + fn get_partial_trie_unwind_marker_rejects_invalid_persisted_marker() { + let marker = + PartialStateTrieUnwindMarker { finish_block_number: 42, partial_state_trie: 42 }; + let error = get_partial_trie_unwind_marker(&MockProvider( + Some(serde_json::to_vec(&marker).unwrap()), + None, + )) + .unwrap_err(); + + assert!(error.to_string().contains("is not below original Finish")); + } + + #[test] + fn get_partial_trie_unwind_marker_rejects_malformed_metadata() { + assert!(get_partial_trie_unwind_marker(&MockProvider(Some(vec![0xff]), None)).is_err()); + } } diff --git a/crates/stages/stages/src/stages/merkle.rs b/crates/stages/stages/src/stages/merkle.rs index bb1e44b6668..5dd05a247d2 100644 --- a/crates/stages/stages/src/stages/merkle.rs +++ b/crates/stages/stages/src/stages/merkle.rs @@ -90,7 +90,10 @@ pub enum MerkleStage { incremental_threshold: u64, }, /// The unwind portion of the merkle stage. - Unwind, + Unwind { + /// Whether every child of a changed branch path should be walked. + walk_all_changed_branch_children: bool, + }, /// Able to execute and unwind. Used for tests #[cfg(any(test, feature = "test-utils"))] Both { @@ -115,7 +118,12 @@ impl MerkleStage { /// Stage default for the [`MerkleStage::Unwind`]. pub const fn default_unwind() -> Self { - Self::Unwind + Self::new_unwind(false) + } + + /// Create a new instance of [`MerkleStage::Unwind`]. + pub const fn new_unwind(walk_all_changed_branch_children: bool) -> Self { + Self::Unwind { walk_all_changed_branch_children } } /// Create new instance of [`MerkleStage::Execution`]. @@ -174,7 +182,7 @@ where fn id(&self) -> StageId { match self { Self::Execution { .. } => StageId::MerkleExecute, - Self::Unwind => StageId::MerkleUnwind, + Self::Unwind { .. } => StageId::MerkleUnwind, #[cfg(any(test, feature = "test-utils"))] Self::Both { .. } => StageId::Other("MerkleBoth"), } @@ -183,7 +191,7 @@ where /// Execute the stage. fn execute(&mut self, provider: &Provider, input: ExecInput) -> Result { let (threshold, incremental_threshold) = match self { - Self::Unwind => { + Self::Unwind { .. } => { info!(target: "sync::stages::merkle::unwind", "Stage is always skipped"); return Ok(ExecOutput::done(StageCheckpoint::new(input.target()))) } @@ -377,6 +385,12 @@ where info!(target: "sync::stages::merkle::unwind", "Stage is always skipped"); return Ok(UnwindOutput { checkpoint: StageCheckpoint::new(input.unwind_to) }) } + let walk_all_changed_branch_children = match self { + Self::Unwind { walk_all_changed_branch_children } => *walk_all_changed_branch_children, + #[cfg(any(test, feature = "test-utils"))] + Self::Both { .. } => false, + Self::Execution { .. } => unreachable!(), + }; let mut entities_checkpoint = input.checkpoint.entities_stage_checkpoint().unwrap_or(EntitiesCheckpoint { @@ -402,7 +416,13 @@ where info!(target: "sync::stages::merkle::unwind", "Nothing to unwind"); } else { let (block_root, updates) = reth_trie_db::with_adapter!(provider, |A| { - DbStateRoot::<_, A>::incremental_root_with_updates(provider, range) + DbStateRoot::<_, A>::incremental_root_calculator(provider, range).and_then( + |calculator| { + calculator + .with_walk_all_changed_branch_children(walk_all_changed_branch_children) + .root_with_updates() + }, + ) }) .map_err(|e| StageError::Fatal(Box::new(e)))?; diff --git a/crates/storage/db-api/src/models/metadata.rs b/crates/storage/db-api/src/models/metadata.rs index 67aa3ea2b6d..025c5e64523 100644 --- a/crates/storage/db-api/src/models/metadata.rs +++ b/crates/storage/db-api/src/models/metadata.rs @@ -91,3 +91,12 @@ impl StorageSettings { self.storage_v2 } } + +/// Marker for an in-progress partial state trie unwind. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +pub struct PartialStateTrieUnwindMarker { + /// The Finish stage block number before the unwind started. + pub finish_block_number: u64, + /// The partial state trie frontier the pipeline is unwinding to. + pub partial_state_trie: u64, +} diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 2accb69ddf1..96d891e37e0 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -4045,6 +4045,11 @@ impl MetadataWriter for DatabaseProvider { fn write_metadata(&self, key: &str, value: Vec) -> ProviderResult<()> { self.tx.put::(key.to_string(), value).map_err(Into::into) } + + fn delete_metadata(&self, key: &str) -> ProviderResult<()> { + self.tx.delete::(key.to_string(), None)?; + Ok(()) + } } impl StorageSettingsCache for DatabaseProvider { @@ -4082,7 +4087,7 @@ mod tests { use reth_ethereum_primitives::Receipt; use reth_execution_types::{AccountRevertInit, BlockExecutionOutput, BlockExecutionResult}; use reth_primitives_traits::SealedBlock; - use reth_storage_api::MetadataWriter; + use reth_storage_api::{MetadataProvider, MetadataWriter}; use reth_testing_utils::generators::{self, random_block, BlockParams}; use reth_trie::{ HashedPostState, KeccakKeyHasher, Nibbles, SortedTrieData, StoredNibbles, @@ -4123,6 +4128,22 @@ mod tests { assert_eq!(result, Vec::>::new()); } + #[test] + fn metadata_can_be_deleted() { + let factory = create_test_provider_factory(); + let key = "metadata-delete-test"; + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.write_metadata(key, vec![1]).unwrap(); + provider_rw.commit().unwrap(); + assert_eq!(factory.provider().unwrap().get_metadata(key).unwrap(), Some(vec![1])); + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw.delete_metadata(key).unwrap(); + provider_rw.commit().unwrap(); + assert_eq!(factory.provider().unwrap().get_metadata(key).unwrap(), None); + } + #[test] fn unwind_commit_waits_for_pre_commit_readers() { let factory = create_test_provider_factory(); diff --git a/crates/storage/storage-api/src/metadata.rs b/crates/storage/storage-api/src/metadata.rs index 927d262fcac..8bfee7e76f2 100644 --- a/crates/storage/storage-api/src/metadata.rs +++ b/crates/storage/storage-api/src/metadata.rs @@ -33,6 +33,11 @@ pub trait MetadataWriter: Send { /// Write a metadata value fn write_metadata(&self, key: &str, value: Vec) -> ProviderResult<()>; + /// Delete a metadata value. + fn delete_metadata(&self, _key: &str) -> ProviderResult<()> { + Err(ProviderError::UnsupportedProvider) + } + /// Write storage settings for this node /// /// Be sure to update provider factory cache with diff --git a/crates/trie/trie/src/trie.rs b/crates/trie/trie/src/trie.rs index 7bc4e83d011..280ad5b7d6c 100644 --- a/crates/trie/trie/src/trie.rs +++ b/crates/trie/trie/src/trie.rs @@ -36,6 +36,8 @@ pub struct StateRoot { pub hashed_cursor_factory: H, /// A set of prefix sets that have changed. pub prefix_sets: TriePrefixSets, + /// Whether every child under a branch whose path matches the prefix set should be walked. + walk_all_changed_branch_children: bool, /// Previous intermediate state. previous_state: Option, /// The number of updates after which the intermediate progress should be returned. @@ -56,6 +58,7 @@ impl StateRoot { trie_cursor_factory, hashed_cursor_factory, prefix_sets: TriePrefixSets::default(), + walk_all_changed_branch_children: false, previous_state: None, threshold: DEFAULT_INTERMEDIATE_THRESHOLD, #[cfg(feature = "metrics")] @@ -69,6 +72,12 @@ impl StateRoot { self } + /// Configures the state root walker to visit all children of changed branch paths. + pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { + self.walk_all_changed_branch_children = enabled; + self + } + /// Set the threshold. pub const fn with_threshold(mut self, threshold: u64) -> Self { self.threshold = threshold; @@ -93,6 +102,7 @@ impl StateRoot { trie_cursor_factory: self.trie_cursor_factory, hashed_cursor_factory, prefix_sets: self.prefix_sets, + walk_all_changed_branch_children: self.walk_all_changed_branch_children, threshold: self.threshold, previous_state: self.previous_state, #[cfg(feature = "metrics")] @@ -106,6 +116,7 @@ impl StateRoot { trie_cursor_factory, hashed_cursor_factory: self.hashed_cursor_factory, prefix_sets: self.prefix_sets, + walk_all_changed_branch_children: self.walk_all_changed_branch_children, threshold: self.threshold, previous_state: self.previous_state, #[cfg(feature = "metrics")] @@ -183,6 +194,7 @@ where account_root_state.walker_stack, self.prefix_sets.account_prefix_set, ) + .with_walk_all_changed_branch_children(self.walk_all_changed_branch_children) .with_deletions_retained(retain_updates); let account_node_iter = TrieNodeIter::state_trie(walker, hashed_account_cursor) .with_last_hashed_key(account_root_state.last_hashed_key); @@ -223,6 +235,7 @@ where .cloned() .unwrap_or_default(), previous_state: Some(storage_state.state), + walk_all_changed_branch_children: self.walk_all_changed_branch_children, threshold: remaining_threshold, retain_updates, }, @@ -254,6 +267,7 @@ where // calculation let hash_builder = HashBuilder::default().with_updates(retain_updates); let walker = TrieWalker::state_trie(trie_cursor, self.prefix_sets.account_prefix_set) + .with_walk_all_changed_branch_children(self.walk_all_changed_branch_children) .with_deletions_retained(retain_updates); let node_iter = TrieNodeIter::state_trie(walker, hashed_account_cursor); (hash_builder, node_iter) @@ -294,6 +308,7 @@ where .cloned() .unwrap_or_default(), previous_state: None, + walk_all_changed_branch_children: self.walk_all_changed_branch_children, threshold: remaining_threshold, retain_updates, }, @@ -494,6 +509,8 @@ pub struct StorageRoot { pub hashed_address: B256, /// The set of storage slot prefixes that have changed. pub prefix_set: PrefixSet, + /// Whether every child under a branch whose path matches the prefix set should be walked. + walk_all_changed_branch_children: bool, /// Previous intermediate state. previous_state: Option, /// The number of updates after which the intermediate progress should be returned. @@ -535,6 +552,7 @@ impl StorageRoot { hashed_cursor_factory, hashed_address, prefix_set, + walk_all_changed_branch_children: false, previous_state: None, threshold: DEFAULT_INTERMEDIATE_THRESHOLD, #[cfg(feature = "metrics")] @@ -548,6 +566,12 @@ impl StorageRoot { self } + /// Configures the storage root walker to visit all children of changed branch paths. + pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { + self.walk_all_changed_branch_children = enabled; + self + } + /// Set the threshold. pub const fn with_threshold(mut self, threshold: u64) -> Self { self.threshold = threshold; @@ -573,6 +597,7 @@ impl StorageRoot { hashed_cursor_factory, hashed_address: self.hashed_address, prefix_set: self.prefix_set, + walk_all_changed_branch_children: self.walk_all_changed_branch_children, previous_state: self.previous_state, threshold: self.threshold, #[cfg(feature = "metrics")] @@ -587,6 +612,7 @@ impl StorageRoot { hashed_cursor_factory: self.hashed_cursor_factory, hashed_address: self.hashed_address, prefix_set: self.prefix_set, + walk_all_changed_branch_children: self.walk_all_changed_branch_children, previous_state: self.previous_state, threshold: self.threshold, #[cfg(feature = "metrics")] @@ -649,6 +675,7 @@ where hashed_cursor_factory, hashed_address, prefix_set, + walk_all_changed_branch_children, previous_state, threshold, #[cfg(feature = "metrics")] @@ -664,6 +691,7 @@ where hashed_address, prefix_set, previous_state, + walk_all_changed_branch_children, threshold, retain_updates, }, @@ -691,6 +719,7 @@ where trie_cursor_factory, hashed_address, prefix_set, + walk_all_changed_branch_children, previous_state, threshold, #[cfg(feature = "metrics")] @@ -705,6 +734,7 @@ where hashed_address, prefix_set, previous_state, + walk_all_changed_branch_children, threshold, retain_updates, }, @@ -731,6 +761,7 @@ where hashed_address, prefix_set, previous_state, + walk_all_changed_branch_children, threshold, retain_updates, } = calculation; @@ -762,6 +793,7 @@ where state.walker_stack, prefix_set, ) + .with_walk_all_changed_branch_children(walk_all_changed_branch_children) .with_deletions_retained(retain_updates); let node_iter = TrieNodeIter::storage_trie(walker, hashed_storage_cursor) .with_last_hashed_key(state.last_hashed_key); @@ -770,6 +802,7 @@ where None => { let hash_builder = HashBuilder::default().with_updates(retain_updates); let walker = TrieWalker::storage_trie(trie_cursor, prefix_set) + .with_walk_all_changed_branch_children(walk_all_changed_branch_children) .with_deletions_retained(retain_updates); let node_iter = TrieNodeIter::storage_trie(walker, hashed_storage_cursor); (hash_builder, node_iter) @@ -847,6 +880,7 @@ struct StorageRootCalculation { hashed_address: B256, prefix_set: PrefixSet, previous_state: Option, + walk_all_changed_branch_children: bool, threshold: u64, retain_updates: bool, } diff --git a/crates/trie/trie/src/walker.rs b/crates/trie/trie/src/walker.rs index f12bf46f748..a8442578575 100644 --- a/crates/trie/trie/src/walker.rs +++ b/crates/trie/trie/src/walker.rs @@ -8,6 +8,18 @@ use alloy_trie::proof::AddedRemovedKeys; use reth_storage_errors::db::DatabaseError; use tracing::{instrument, trace}; +#[cfg(test)] +use crate::trie_cursor::{mock::MockTrieCursorFactory, TrieCursorFactory}; + +#[cfg(test)] +use alloy_primitives::map::B256Map; + +#[cfg(test)] +use alloy_trie::TrieMask; + +#[cfg(test)] +use std::collections::BTreeMap; + #[cfg(feature = "metrics")] use crate::metrics::WalkerMetrics; @@ -26,6 +38,9 @@ pub struct TrieWalker { pub can_skip_current_node: bool, /// A `PrefixSet` representing the changes to be applied to the trie. pub changes: PrefixSet, + /// When enabled, all children of a branch become unskippable if the branch path itself + /// matches the prefix set, even if a given child path does not. + walk_all_changed_branch_children: bool, /// The retained trie node keys that need to be removed. removed_keys: Option>, /// Provided when it's necessary not to skip certain nodes during proof generation. @@ -76,6 +91,7 @@ impl> TrieWalker { changes, stack, can_skip_current_node: false, + walk_all_changed_branch_children: false, removed_keys: None, added_removed_keys: None, #[cfg(feature = "metrics")] @@ -101,6 +117,7 @@ impl> TrieWalker { stack: self.stack, can_skip_current_node: self.can_skip_current_node, changes: self.changes, + walk_all_changed_branch_children: self.walk_all_changed_branch_children, removed_keys: self.removed_keys, added_removed_keys, #[cfg(feature = "metrics")] @@ -108,6 +125,12 @@ impl> TrieWalker { } } + /// Configures the walker to treat every child of a matching branch path as unskippable. + pub const fn with_walk_all_changed_branch_children(mut self, enabled: bool) -> Self { + self.walk_all_changed_branch_children = enabled; + self + } + /// Split the walker into stack and trie updates. pub fn split(mut self) -> (Vec, HashSet) { let keys = self.take_removed_keys(); @@ -188,7 +211,12 @@ impl> TrieWalker { "Checked for only non-removed child", ); + let branch_path_matches_prefix_set = self.walk_all_changed_branch_children && + node.position().is_child() && + self.changes.contains(&node.key); + !self.changes.contains(node.full_key()) && + !branch_path_matches_prefix_set && node.hash_flag() && !key_is_only_nonremoved_child }); @@ -233,6 +261,7 @@ impl> TrieWalker { changes, stack: vec![CursorSubNode::default()], can_skip_current_node: false, + walk_all_changed_branch_children: false, removed_keys: None, added_removed_keys: Default::default(), #[cfg(feature = "metrics")] @@ -387,3 +416,83 @@ impl> TrieWalker { Ok(()) } } + +#[cfg(test)] +mod tests { + use super::*; + use crate::prefix_set::PrefixSetMut; + use alloy_primitives::B256; + + fn branch_node(state_mask: u16, tree_mask: u16, hash_mask: u16) -> BranchNodeCompact { + let hash_count = hash_mask.count_ones() as usize; + BranchNodeCompact::new( + TrieMask::new(state_mask), + TrieMask::new(tree_mask), + TrieMask::new(hash_mask), + vec![B256::ZERO; hash_count], + None, + ) + } + + fn root_branch_node(state_mask: u16, tree_mask: u16, hash_mask: u16) -> BranchNodeCompact { + let hash_count = hash_mask.count_ones() as usize; + BranchNodeCompact::new( + TrieMask::new(state_mask), + TrieMask::new(tree_mask), + TrieMask::new(hash_mask), + vec![B256::ZERO; hash_count], + Some(B256::ZERO), + ) + } + + fn walker_for_matching_branch_children_test( + walk_all_changed_branch_children: bool, + ) -> TrieWalker { + let trie_nodes = BTreeMap::from([ + (Nibbles::default(), root_branch_node(1 << 2, 1 << 2, 1 << 2)), + ( + Nibbles::from_nibbles([0x2]), + branch_node((1 << 3) | (1 << 4), 0, (1 << 3) | (1 << 4)), + ), + ]); + let factory = MockTrieCursorFactory::new(trie_nodes, B256Map::default()); + + let mut prefix_set = PrefixSetMut::default(); + prefix_set.insert(Nibbles::from_nibbles([0x2, 0x3, 0x1])); + + TrieWalker::state_trie(factory.account_trie_cursor().unwrap(), prefix_set.freeze()) + .with_walk_all_changed_branch_children(walk_all_changed_branch_children) + } + + #[test] + fn branch_siblings_remain_skippable_by_default() { + let mut walker = walker_for_matching_branch_children_test(false); + + assert_eq!(walker.key().copied(), Some(Nibbles::default())); + assert!(!walker.can_skip_current_node); + + walker.advance().unwrap(); + assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2]))); + assert!(!walker.can_skip_current_node); + + walker.advance().unwrap(); + assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x3]))); + assert_eq!(walker.stack.last().unwrap().position(), SubNodePosition::Child(0x3)); + assert!(!walker.can_skip_current_node); + + walker.advance().unwrap(); + assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x4]))); + assert!(walker.can_skip_current_node); + } + + #[test] + fn matching_branch_path_can_make_all_children_unskippable() { + let mut walker = walker_for_matching_branch_children_test(true); + + walker.advance().unwrap(); + walker.advance().unwrap(); + walker.advance().unwrap(); + assert_eq!(walker.key().copied(), Some(Nibbles::from_nibbles([0x2, 0x4]))); + assert!(!walker.can_skip_current_node); + } +} From b3112af5aa7174298546ec6e66410b7e10d0820e Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 4 Aug 2026 17:28:51 +0200 Subject: [PATCH 089/148] feat(engine): build payloads on canonical ancestors above finality (#26567) --- .../src/testsuite/actions/mod.rs | 6 +- .../src/testsuite/actions/produce_blocks.rs | 5 +- crates/engine/primitives/src/message.rs | 9 ++ crates/engine/tree/src/tree/mod.rs | 60 ++++---- crates/engine/tree/src/tree/tests.rs | 132 ++++++++++++++++++ .../e2e-testsuite/fcu_finalized_blocks.rs | 53 +++---- crates/rpc/rpc-engine-api/src/error.rs | 18 ++- 7 files changed, 231 insertions(+), 52 deletions(-) diff --git a/crates/e2e-test-utils/src/testsuite/actions/mod.rs b/crates/e2e-test-utils/src/testsuite/actions/mod.rs index d4916265692..28edd360091 100644 --- a/crates/e2e-test-utils/src/testsuite/actions/mod.rs +++ b/crates/e2e-test-utils/src/testsuite/actions/mod.rs @@ -1,6 +1,7 @@ //! Actions that can be performed in tests. use crate::testsuite::Environment; +use alloy_primitives::B256; use alloy_rpc_types_engine::{ForkchoiceState, ForkchoiceUpdated, PayloadStatusEnum}; use eyre::Result; use futures_util::future::BoxFuture; @@ -145,7 +146,10 @@ where let fork_choice_state = ForkchoiceState { head_block_hash: latest_block.hash, safe_block_hash: latest_block.hash, - finalized_block_hash: latest_block.hash, + // Making a block canonical does not imply finality: tests advance the + // finalized block explicitly via `FinalizeBlock`, and a finalized tip would + // reject any later forkchoice update below it as a too deep reorg. + finalized_block_hash: B256::ZERO, }; let active_idx = env.active_node_idx; diff --git a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs index b28ea0f19ae..39d3af4b626 100644 --- a/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs +++ b/crates/e2e-test-utils/src/testsuite/actions/produce_blocks.rs @@ -402,7 +402,10 @@ where let fork_choice_state = ForkchoiceState { head_block_hash: head_hash, safe_block_hash: head_hash, - finalized_block_hash: head_hash, + // Making a block canonical does not imply finality: tests advance the finalized + // block explicitly via `FinalizeBlock`, and a finalized tip would reject any + // later forkchoice update below it as a too deep reorg. + finalized_block_hash: B256::ZERO, }; debug!( "Broadcasting forkchoice update to {} clients. Head: {:?}", diff --git a/crates/engine/primitives/src/message.rs b/crates/engine/primitives/src/message.rs index c57edf64c60..9e7c70fab76 100644 --- a/crates/engine/primitives/src/message.rs +++ b/crates/engine/primitives/src/message.rs @@ -83,6 +83,15 @@ impl OnForkChoiceUpdated { } } + /// Creates a new instance of `OnForkChoiceUpdated` if the forkchoice update failed because the + /// requested reorg to the head block exceeds the supported reorg depth. + pub fn too_deep_reorg() -> Self { + Self { + forkchoice_status: ForkchoiceStatus::Invalid, + fut: Either::Left(futures::future::ready(Err(ForkchoiceUpdateError::TooDeepReorg))), + } + } + /// Creates a new instance of `OnForkChoiceUpdated` if the forkchoice update was successful but /// payload attributes were invalid. pub fn invalid_payload_attributes() -> Self { diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index a1066a1160f..34a85de3e9b 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -1355,34 +1355,46 @@ where // For OpStack, or if explicitly configured, the proposers are allowed to reorg their // own chain at will, so we need to always trigger a new payload job if requested. - if self.engine_kind.is_opstack() || - self.config.always_process_payload_attributes_on_canonical_head() + let always_trigger_payload_job = self.engine_kind.is_opstack() || + self.config.always_process_payload_attributes_on_canonical_head(); + + // A canonical ancestor below the latest known finalized block can never become the + // head again, because this would reorg out the finalized block. Such a forkchoice + // update exceeds the supported reorg depth and is rejected regardless of the payload + // attributes: + // + // + // The stored finalized block is used because a forkchoice update MAY carry a zero + // finalized hash without clearing previously established finality. + if !always_trigger_payload_job && + self.canonical_in_memory_state + .get_finalized_num_hash() + .is_some_and(|finalized| canonical_header.number() < finalized.number) { - // We need to effectively unwind the _canonical_ chain to the FCU's head, which is - // part of the canonical chain. We need to update the latest block state to reflect - // the canonical ancestor. This ensures that state providers and the transaction - // pool operate with the correct chain state after forkchoice update processing, and - // new payloads built on the reorg'd head will be added to the tree immediately. - if self.config.unwind_canonical_header() { - self.update_latest_block_to_canonical_ancestor(&canonical_header)?; - } + debug!(target: "engine::tree", head = canonical_header.number(), "rejecting canonical ancestor fcu below the finalized block"); + return Ok(Some(TreeOutcome::new(OnForkChoiceUpdated::too_deep_reorg()))); + } - if let Some(attr) = attrs { - debug!(target: "engine::tree", head = canonical_header.number(), "handling payload attributes for canonical head"); - // Clone only when we actually need to process the attributes - let updated = - self.process_payload_attributes(attr.clone(), &canonical_header, state); - return Ok(Some(TreeOutcome::new(updated))); - } + // We need to effectively unwind the _canonical_ chain to the FCU's head, which is + // part of the canonical chain. We need to update the latest block state to reflect + // the canonical ancestor. This ensures that state providers and the transaction + // pool operate with the correct chain state after forkchoice update processing, and + // new payloads built on the reorg'd head will be added to the tree immediately. + if always_trigger_payload_job && self.config.unwind_canonical_header() { + self.update_latest_block_to_canonical_ancestor(&canonical_header)?; } - // According to the Engine API specification, client software MAY skip an update of the - // forkchoice state and MUST NOT begin a payload build process if - // `forkchoiceState.headBlockHash` references a `VALID` ancestor of the head - // of canonical chain, i.e. the ancestor passed payload validation process - // and deemed `VALID`. In the case of such an event, client software MUST - // return `{payloadStatus: {status: VALID, latestValidHash: - // forkchoiceState.headBlockHash, validationError: null}, payloadId: null}` + // A canonical ancestor at or above the latest known finalized block can become the + // parent of the next block, e.g. when the CL wants to reorg out the current head. + // The canonical chain remains untouched here; the block built on the ancestor + // triggers the actual reorg once it is inserted via newPayload and FCU'd. + if let Some(attr) = attrs { + debug!(target: "engine::tree", head = canonical_header.number(), "handling payload attributes for canonical head"); + // Clone only when we actually need to process the attributes + let updated = + self.process_payload_attributes(attr.clone(), &canonical_header, state); + return Ok(Some(TreeOutcome::new(updated))); + } // The head block is already canonical and we're not processing payload attributes, // so we're not triggering a payload job and can return right away diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 6ded5d35e5c..53cbf1d4e5f 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -17,6 +17,7 @@ use alloy_primitives::{ use alloy_rlp::Decodable; use alloy_rpc_types_engine::{ ExecutionData, ExecutionPayloadSidecar, ExecutionPayloadV1, ForkchoiceState, + ForkchoiceUpdateError, }; use assert_matches::assert_matches; use reth_chain_state::{test_utils::TestBlockBuilder, BlockState}; @@ -1658,6 +1659,137 @@ async fn test_fcu_with_canonical_ancestor_updates_latest_block() { ); } +#[tokio::test] +async fn test_fcu_with_canonical_ancestor_below_finalized_is_rejected() { + reth_tracing::init_test_tracing(); + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec.clone()); + let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..5).collect(); + test_harness = test_harness.with_blocks(blocks.clone()); + + let current_head = blocks[3].recovered_block(); + let finalized = blocks[2].recovered_block(); + let ancestor = blocks[1].recovered_block(); + test_harness.tree.canonical_in_memory_state.set_finalized(finalized.clone_sealed_header()); + + // An FCU to an ancestor below the latest known finalized block would reorg out the + // finalized block and is rejected, with or without payload attributes. + let payload_attributes = EthPayloadAttributes { + timestamp: ancestor.timestamp() + 1, + prev_randao: B256::ZERO, + suggested_fee_recipient: Default::default(), + withdrawals: None, + parent_beacon_block_root: None, + slot_number: None, + target_gas_limit: None, + }; + for attrs in [Some(payload_attributes), None] { + let err = test_harness + .tree + .on_forkchoice_updated( + ForkchoiceState { + head_block_hash: ancestor.hash(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }, + attrs, + ) + .unwrap() + .outcome + .await + .unwrap_err(); + + assert_matches!(err, ForkchoiceUpdateError::TooDeepReorg); + } + + // no payload build is started and the canonical head remains untouched + assert!(test_harness.payload_command_rx.try_recv().is_err()); + assert_eq!(test_harness.tree.state.tree_state.canonical_block_hash(), current_head.hash()); + + // the finalized block itself is not below finality and can become the parent of the next + // block + let payload_attributes = EthPayloadAttributes { + timestamp: finalized.timestamp() + 1, + prev_randao: B256::ZERO, + suggested_fee_recipient: Default::default(), + withdrawals: None, + parent_beacon_block_root: None, + slot_number: None, + target_gas_limit: None, + }; + let outcome = test_harness + .tree + .on_forkchoice_updated( + ForkchoiceState { + head_block_hash: finalized.hash(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }, + Some(payload_attributes), + ) + .unwrap(); + + assert_eq!(outcome.outcome.forkchoice_status(), ForkchoiceStatus::Valid); + let command = test_harness.payload_command_rx.try_recv().unwrap(); + let PayloadServiceCommand::BuildNewPayload(input, _, _) = command else { + panic!("expected build new payload command") + }; + assert_eq!(input.parent_hash, finalized.hash()); +} + +#[tokio::test] +async fn test_fcu_with_canonical_ancestor_above_finalized_starts_payload_build() { + reth_tracing::init_test_tracing(); + let chain_spec = MAINNET.clone(); + let mut test_harness = TestHarness::new(chain_spec.clone()); + let mut test_block_builder = TestBlockBuilder::eth().with_chain_spec((*chain_spec).clone()); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..5).collect(); + test_harness = test_harness.with_blocks(blocks.clone()); + + let current_head = blocks[3].recovered_block(); + let finalized = blocks[0].recovered_block(); + let ancestor = blocks[2].recovered_block(); + test_harness.tree.canonical_in_memory_state.set_finalized(finalized.clone_sealed_header()); + + let payload_attributes = EthPayloadAttributes { + timestamp: ancestor.timestamp() + 1, + prev_randao: B256::ZERO, + suggested_fee_recipient: Default::default(), + withdrawals: None, + parent_beacon_block_root: None, + slot_number: None, + target_gas_limit: None, + }; + let outcome = test_harness + .tree + .on_forkchoice_updated( + ForkchoiceState { + head_block_hash: ancestor.hash(), + safe_block_hash: B256::ZERO, + finalized_block_hash: B256::ZERO, + }, + Some(payload_attributes), + ) + .unwrap(); + + assert_eq!(outcome.outcome.forkchoice_status(), ForkchoiceStatus::Valid); + + // a payload build is started on top of the ancestor + let command = test_harness.payload_command_rx.try_recv().unwrap(); + let PayloadServiceCommand::BuildNewPayload(input, _, _) = command else { + panic!("expected build new payload command") + }; + assert_eq!(input.parent_hash, ancestor.hash()); + + // the canonical chain remains untouched, only the built block reorgs it eventually + assert_eq!(test_harness.tree.state.tree_state.canonical_block_hash(), current_head.hash()); + assert_eq!( + test_harness.tree.canonical_in_memory_state.get_canonical_head().hash(), + current_head.hash() + ); +} + /// Test that verifies the happy path where a new payload extends the canonical chain #[test] fn test_on_new_payload_canonical_insertion() { diff --git a/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs b/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs index 8cafe8b0c1b..2fd579d55f4 100644 --- a/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs +++ b/crates/engine/tree/tests/e2e-testsuite/fcu_finalized_blocks.rs @@ -1,13 +1,11 @@ -//! E2E test for forkchoice update with finalized blocks. -//! -//! This test verifies the behavior when attempting to reorg behind a finalized block. +//! E2E tests for forkchoice updates to canonical ancestors around the finalized block. use eyre::Result; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::testsuite::{ actions::{ - BlockReference, CaptureBlock, CreateFork, FinalizeBlock, MakeCanonical, ProduceBlocks, - SendForkchoiceUpdate, + AssertChainTip, BlockReference, CaptureBlock, CreateFork, FinalizeBlock, MakeCanonical, + ProduceBlocks, SendForkchoiceUpdate, UpdateBlockInfo, }, setup::{NetworkSetup, Setup}, TestBuilder, @@ -36,40 +34,45 @@ fn default_engine_tree_setup() -> Setup { .with_tree_config(TreeConfig::default().with_has_enough_parallelism(true)) } -/// This test: -/// 1. Creates a main chain and finalizes a block -/// 2. Creates a fork that branches BEFORE the finalized block -/// 3. Attempts to switch to that fork (which would require changing history behind finalized) +/// Verifies that an FCU to a canonical ancestor above finality can start a payload build on top +/// of the ancestor whose block eventually reorgs out the current head. #[tokio::test] -async fn test_reorg_to_fork_behind_finalized() -> Result<()> { +async fn test_fcu_to_canonical_ancestor_around_finalized() -> Result<()> { reth_tracing::init_test_tracing(); let test = TestBuilder::new() .with_setup(default_engine_tree_setup()) - // Build main chain: blocks 1-10 - .with_action(ProduceBlocks::::new(10)) + // Build and tag canonical ancestors on the way to block 10. + .with_action(ProduceBlocks::::new(7)) + .with_action(CaptureBlock::new("block_7")) + .with_action(ProduceBlocks::::new(1)) + .with_action(CaptureBlock::new("block_8")) + .with_action(ProduceBlocks::::new(2)) + .with_action(CaptureBlock::new("block_10")) .with_action(MakeCanonical::new()) - // Capture blocks for the test - .with_action(CaptureBlock::new("block_5")) // Will be fork point - .with_action(CaptureBlock::new("block_7")) // Will be finalized - .with_action(CaptureBlock::new("block_10")) // Current head - // Create a fork from block 5 (before block 7 which will be finalized) - .with_action(CreateFork::::new(5, 5)) // Fork from block 5, add 5 blocks - .with_action(CaptureBlock::new("fork_tip")) - // Step 1: Finalize block 7 with head at block 10 + // Establish block 7 as the latest known finalized block. .with_action( FinalizeBlock::::new(BlockReference::Tag("block_7".to_string())) .with_head(BlockReference::Tag("block_10".to_string())), ) - // Step 2: Attempt to reorg to a fork that doesn't contain the finalized block + // Block 8 is above finality: without payload attributes the FCU is acknowledged, but the + // canonical head does not move. .with_action( SendForkchoiceUpdate::::new( - BlockReference::Tag("block_7".to_string()), // Keep finalized - BlockReference::Tag("fork_tip".to_string()), // New safe - BlockReference::Tag("fork_tip".to_string()), // New head + BlockReference::Tag("block_7".to_string()), + BlockReference::Tag("block_7".to_string()), + BlockReference::Tag("block_8".to_string()), ) .with_expected_status(alloy_rpc_types_engine::PayloadStatusEnum::Valid), - ); + ) + .with_action(UpdateBlockInfo::default()) + .with_action(AssertChainTip::new(10)) + // With payload attributes, an FCU to block 8 starts a payload build on top of it: + // `CreateFork` drives the full fcu(attrs) -> getPayload -> newPayload flow. Making the + // built block canonical reorgs out the previous blocks 9 and 10. + .with_action(CreateFork::::new_from_tag("block_8", 1)) + .with_action(MakeCanonical::new()) + .with_action(AssertChainTip::new(9)); test.run::().await?; diff --git a/crates/rpc/rpc-engine-api/src/error.rs b/crates/rpc/rpc-engine-api/src/error.rs index f6070b0515a..3c7fd2c8a66 100644 --- a/crates/rpc/rpc-engine-api/src/error.rs +++ b/crates/rpc/rpc-engine-api/src/error.rs @@ -1,7 +1,8 @@ use alloy_primitives::B256; use alloy_rpc_types_engine::{ ForkchoiceUpdateError, INVALID_FORK_CHOICE_STATE_ERROR, INVALID_FORK_CHOICE_STATE_ERROR_MSG, - INVALID_PAYLOAD_ATTRIBUTES_ERROR, INVALID_PAYLOAD_ATTRIBUTES_ERROR_MSG, + INVALID_PAYLOAD_ATTRIBUTES_ERROR, INVALID_PAYLOAD_ATTRIBUTES_ERROR_MSG, TOO_DEEP_REORG_ERROR, + TOO_DEEP_REORG_ERROR_MSG, }; use jsonrpsee_types::error::{ INTERNAL_ERROR_CODE, INVALID_PARAMS_CODE, INVALID_PARAMS_MSG, SERVER_ERROR_MSG, @@ -195,6 +196,13 @@ impl From for jsonrpsee_types::error::ErrorObject<'static> { None::<()>, ) } + ForkchoiceUpdateError::TooDeepReorg => { + jsonrpsee_types::error::ErrorObject::owned( + TOO_DEEP_REORG_ERROR, + TOO_DEEP_REORG_ERROR_MSG, + None::<()>, + ) + } // Map future alloy forkchoice errors as internal until handled. #[allow(unreachable_patterns, clippy::needless_return)] _ => { @@ -279,6 +287,14 @@ mod tests { )), ); + ensure_engine_rpc_error( + -38006, + "Too deep reorg", + EngineApiError::ForkChoiceUpdate(BeaconForkChoiceUpdateError::ForkchoiceUpdateError( + ForkchoiceUpdateError::TooDeepReorg, + )), + ); + ensure_engine_rpc_error( UNKNOWN_PAYLOAD_CODE, "Unknown payload", From d56b19823b47f2e336ac0b86cd642081dc613a6e Mon Sep 17 00:00:00 2001 From: Francis Li Date: Tue, 4 Aug 2026 09:06:10 -0700 Subject: [PATCH 090/148] feat(rpc): support network-specific log responses (#26491) Co-authored-by: Amp Co-authored-by: Arsenii Kulikov Co-authored-by: Matthias Seitz --- crates/rpc/rpc-api/src/engine.rs | 6 +- crates/rpc/rpc-builder/tests/it/http.rs | 23 +++--- crates/rpc/rpc-convert/src/rpc.rs | 7 ++ crates/rpc/rpc-convert/src/transaction.rs | 65 ++++++++-------- crates/rpc/rpc-eth-api/src/filter.rs | 14 ++-- crates/rpc/rpc-eth-api/src/helpers/block.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/receipt.rs | 31 +++++--- .../rpc-eth-api/src/helpers/subscriptions.rs | 52 +++++++++---- .../rpc-eth-api/src/helpers/transaction.rs | 14 ++-- crates/rpc/rpc-eth-types/src/block.rs | 2 +- crates/rpc/rpc-eth-types/src/logs_utils.rs | 51 +++++++----- crates/rpc/rpc-eth-types/src/receipt.rs | 13 +++- crates/rpc/rpc/src/engine.rs | 10 ++- crates/rpc/rpc/src/eth/filter.rs | 77 +++++++++++-------- crates/rpc/rpc/src/eth/pubsub.rs | 7 +- 15 files changed, 230 insertions(+), 144 deletions(-) diff --git a/crates/rpc/rpc-api/src/engine.rs b/crates/rpc/rpc-api/src/engine.rs index e3714d6ecf1..61256141070 100644 --- a/crates/rpc/rpc-api/src/engine.rs +++ b/crates/rpc/rpc-api/src/engine.rs @@ -16,7 +16,7 @@ use alloy_rpc_types_engine::{ PayloadId, PayloadStatus, }; use alloy_rpc_types_eth::{ - state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, Log, SyncStatus, + state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, SyncStatus, }; use alloy_serde::JsonStorageKey; use jsonrpsee::{core::RpcResult, proc_macros::rpc, RpcModule}; @@ -361,7 +361,7 @@ pub trait EngineApi { /// Specifically for the engine auth server: #[cfg_attr(not(feature = "client"), rpc(server, namespace = "eth"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "eth"))] -pub trait EngineEthApi { +pub trait EngineEthApi { /// Returns an object with data about the sync status or false. #[method(name = "syncing")] fn syncing(&self) -> RpcResult; @@ -410,7 +410,7 @@ pub trait EngineEthApi { /// Returns logs matching given filter object. #[method(name = "getLogs")] - async fn logs(&self, filter: Filter) -> RpcResult>; + async fn logs(&self, filter: Filter) -> RpcResult>; /// Returns the account and storage values of the specified account including the Merkle-proof. /// This call can be used to verify that the data you are pulling from is not tampered with. diff --git a/crates/rpc/rpc-builder/tests/it/http.rs b/crates/rpc/rpc-builder/tests/it/http.rs index 1012b86d895..5160ce3020f 100644 --- a/crates/rpc/rpc-builder/tests/it/http.rs +++ b/crates/rpc/rpc-builder/tests/it/http.rs @@ -134,21 +134,24 @@ async fn test_filter_calls(client: &C) where C: ClientT + SubscriptionClientT + Sync, { - EthFilterApiClient::::new_filter(client, Filter::default()).await.unwrap(); - EthFilterApiClient::::new_pending_transaction_filter(client, None).await.unwrap(); - EthFilterApiClient::::new_pending_transaction_filter( + EthFilterApiClient::::new_filter(client, Filter::default()).await.unwrap(); + EthFilterApiClient::::new_pending_transaction_filter(client, None) + .await + .unwrap(); + EthFilterApiClient::::new_pending_transaction_filter( client, Some(PendingTransactionFilterKind::Full), ) .await .unwrap(); - let id = EthFilterApiClient::::new_block_filter(client).await.unwrap(); - EthFilterApiClient::::filter_changes(client, id.clone()).await.unwrap(); - EthFilterApiClient::::logs(client, Filter::default()).await.unwrap(); - let id = - EthFilterApiClient::::new_filter(client, Filter::default()).await.unwrap(); - EthFilterApiClient::::filter_logs(client, id.clone()).await.unwrap(); - EthFilterApiClient::::uninstall_filter(client, id).await.unwrap(); + let id = EthFilterApiClient::::new_block_filter(client).await.unwrap(); + EthFilterApiClient::::filter_changes(client, id.clone()).await.unwrap(); + EthFilterApiClient::::logs(client, Filter::default()).await.unwrap(); + let id = EthFilterApiClient::::new_filter(client, Filter::default()) + .await + .unwrap(); + EthFilterApiClient::::filter_logs(client, id.clone()).await.unwrap(); + EthFilterApiClient::::uninstall_filter(client, id).await.unwrap(); } async fn test_basic_admin_calls(client: &C) diff --git a/crates/rpc/rpc-convert/src/rpc.rs b/crates/rpc/rpc-convert/src/rpc.rs index b1425165f7a..dd640a5379b 100644 --- a/crates/rpc/rpc-convert/src/rpc.rs +++ b/crates/rpc/rpc-convert/src/rpc.rs @@ -12,6 +12,8 @@ pub trait RpcTypes: Send + Sync + Clone + Unpin + Debug + 'static { type Header: RpcObject + HeaderResponse; /// Receipt response type. type Receipt: RpcObject + ReceiptResponse; + /// Log response type. + type Log: RpcObject; /// Transaction response type. type TransactionResponse: RpcObject + TransactionResponse; /// Transaction response type. @@ -24,6 +26,8 @@ where { type Header = T::HeaderResponse; type Receipt = T::ReceiptResponse; + // Default network implementations to Ethereum logs until Alloy exposes a log response type. + type Log = alloy_rpc_types_eth::Log; type TransactionResponse = T::TransactionResponse; type TransactionRequest = T::TransactionRequest; } @@ -34,6 +38,9 @@ pub type RpcTransaction = ::TransactionResponse; /// Adapter for network specific receipt response. pub type RpcReceipt = ::Receipt; +/// Adapter for network specific log response. +pub type RpcLog = ::Log; + /// Adapter for network specific header response. pub type RpcHeader = ::Header; diff --git a/crates/rpc/rpc-convert/src/transaction.rs b/crates/rpc/rpc-convert/src/transaction.rs index 980fbeca24f..4b010776a2f 100644 --- a/crates/rpc/rpc-convert/src/transaction.rs +++ b/crates/rpc/rpc-convert/src/transaction.rs @@ -1,10 +1,11 @@ //! Compatibility functions for rpc `Transaction` type. use crate::{ - RpcHeader, RpcReceipt, RpcTransaction, RpcTxReq, RpcTypes, SignableTxRequest, TryIntoTxEnv, + RpcHeader, RpcLog, RpcReceipt, RpcTransaction, RpcTxReq, RpcTypes, SignableTxRequest, + TryIntoTxEnv, }; use alloy_consensus::{error::ValueError, transaction::Recovered}; use alloy_primitives::Address; -use alloy_rpc_types_eth::TransactionInfo; +use alloy_rpc_types_eth::{Log, TransactionInfo}; use core::error; use dyn_clone::DynClone; use reth_evm::{BlockEnvFor, ConfigureEvm, EvmEnvFor, SpecFor, TxEnvFor}; @@ -32,28 +33,29 @@ pub struct ConvertReceiptInput<'a, N: NodePrimitives> { /// A type that knows how to convert primitive receipts to RPC representations. pub trait ReceiptConverter: Debug + 'static { - /// RPC representation. + /// RPC receipt representation. type RpcReceipt; + /// RPC log representation. + type RpcLog; + /// Error that may occur during conversion. type Error; - /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all - /// receipts are from the same block. - fn convert_receipts( + /// Converts an RPC log using its primitive receipt and block header. + fn convert_log( &self, - receipts: Vec>, - ) -> Result, Self::Error>; + log: Log, + receipt: &N::Receipt, + header: &SealedHeaderFor, + ) -> Result; - /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all - /// receipts are from `block`. - fn convert_receipts_with_block( + /// Converts primitive receipts from `block` to RPC representations. + fn convert_receipts( &self, receipts: Vec>, - _block: &SealedBlock, - ) -> Result, Self::Error> { - self.convert_receipts(receipts) - } + block: &SealedBlock, + ) -> Result, Self::Error>; } /// A type that knows how to convert a consensus header into an RPC header. @@ -160,18 +162,16 @@ pub trait RpcConvert: Send + Sync + Unpin + Debug + DynClone + 'static { evm_env: &EvmEnvFor, ) -> Result, Self::Error>; - /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all - /// receipts are from the same block. - fn convert_receipts( + /// Converts an RPC log using its primitive receipt and block header. + fn convert_log( &self, - receipts: Vec>, - ) -> Result>, Self::Error>; + log: Log, + receipt: &::Receipt, + header: &SealedHeaderFor, + ) -> Result, Self::Error>; - /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all - /// receipts are from the same block. - /// - /// Also accepts the corresponding block in case the receipt requires additional metadata. - fn convert_receipts_with_block( + /// Converts primitive receipts from `block` to RPC representations. + fn convert_receipts( &self, receipts: Vec>, block: &SealedBlock>, @@ -677,6 +677,7 @@ where Receipt: ReceiptConverter< N, RpcReceipt = RpcReceipt, + RpcLog = RpcLog, Error: From + From + From<>>::Err> @@ -733,19 +734,21 @@ where self.tx_env_converter.convert_tx_env(request, evm_env).map_err(Into::into) } - fn convert_receipts( + fn convert_log( &self, - receipts: Vec>, - ) -> Result>, Self::Error> { - self.receipt_converter.convert_receipts(receipts) + log: Log, + receipt: &::Receipt, + header: &SealedHeaderFor, + ) -> Result, Self::Error> { + self.receipt_converter.convert_log(log, receipt, header) } - fn convert_receipts_with_block( + fn convert_receipts( &self, receipts: Vec>, block: &SealedBlock>, ) -> Result>, Self::Error> { - self.receipt_converter.convert_receipts_with_block(receipts, block) + self.receipt_converter.convert_receipts(receipts, block) } fn convert_header( diff --git a/crates/rpc/rpc-eth-api/src/filter.rs b/crates/rpc/rpc-eth-api/src/filter.rs index f6de812b3f5..71ade5fa93d 100644 --- a/crates/rpc/rpc-eth-api/src/filter.rs +++ b/crates/rpc/rpc-eth-api/src/filter.rs @@ -1,14 +1,14 @@ //! `eth_` RPC API for filtering. use alloy_json_rpc::RpcObject; -use alloy_rpc_types_eth::{Filter, FilterChanges, FilterId, Log, PendingTransactionFilterKind}; +use alloy_rpc_types_eth::{Filter, FilterChanges, FilterId, PendingTransactionFilterKind}; use jsonrpsee::{core::RpcResult, proc_macros::rpc}; use std::future::Future; /// Rpc Interface for poll-based ethereum filter API. #[cfg_attr(not(feature = "client"), rpc(server, namespace = "eth"))] #[cfg_attr(feature = "client", rpc(server, client, namespace = "eth"))] -pub trait EthFilterApi { +pub trait EthFilterApi { /// Creates a new filter and returns its id. #[method(name = "newFilter")] async fn new_filter(&self, filter: Filter) -> RpcResult; @@ -26,11 +26,11 @@ pub trait EthFilterApi { /// Returns all filter changes since last poll. #[method(name = "getFilterChanges")] - async fn filter_changes(&self, id: FilterId) -> RpcResult>; + async fn filter_changes(&self, id: FilterId) -> RpcResult>; /// Returns all logs matching given filter (in a range 'from' - 'to'). #[method(name = "getFilterLogs")] - async fn filter_logs(&self, id: FilterId) -> RpcResult>; + async fn filter_logs(&self, id: FilterId) -> RpcResult>; /// Uninstalls filter. #[method(name = "uninstallFilter")] @@ -38,7 +38,7 @@ pub trait EthFilterApi { /// Returns logs matching given filter object. #[method(name = "getLogs")] - async fn logs(&self, filter: Filter) -> RpcResult>; + async fn logs(&self, filter: Filter) -> RpcResult>; } /// Limits for logs queries @@ -59,11 +59,11 @@ impl QueryLimits { /// Rpc Interface for poll-based ethereum filter API, implementing only the `eth_getLogs` method. /// Used for the engine API, with possibility to specify [`QueryLimits`]. -pub trait EngineEthFilter: Send + Sync + 'static { +pub trait EngineEthFilter: Send + Sync + 'static { /// Returns logs matching given filter object. fn logs( &self, filter: Filter, limits: QueryLimits, - ) -> impl Future>> + Send; + ) -> impl Future>> + Send; } diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 479b45623c4..0d1d4305897 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -136,7 +136,7 @@ pub trait EthBlocks: LoadBlock, all_receipts: Option>>>, + block: Option>>>, ) -> impl Future, Self::Error>> + Send { async move { let hash = meta.block_hash; + let block = match block { + Some(block) => block, + None => self + .cache() + .get_recovered_block(hash) + .await + .map_err(Self::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(hash.into()))?, + }; // Use pre-fetched receipts if available, otherwise fetch from cache. let all_receipts = match all_receipts { Some(receipts) => receipts, @@ -47,13 +57,16 @@ pub trait LoadReceipt: Ok(self .converter() - .convert_receipts(vec![ConvertReceiptInput { - tx: tx.as_recovered_ref(), - gas_used: receipt.cumulative_gas_used() - gas_used, - receipt, - next_log_index, - meta, - }])? + .convert_receipts( + vec![ConvertReceiptInput { + tx: tx.as_recovered_ref(), + gas_used: receipt.cumulative_gas_used() - gas_used, + receipt, + next_log_index, + meta, + }], + block.sealed_block(), + )? .pop() .unwrap()) } diff --git a/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs b/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs index 97cf9ecb486..51b93d3b7e2 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs @@ -1,8 +1,8 @@ //! Streams subscriptions providers for `eth_subscribe`. -use crate::{EthApiTypes, RpcConvert, RpcNodeCore, RpcReceipt}; +use crate::{EthApiTypes, RpcConvert, RpcLog, RpcNodeCore, RpcReceipt}; use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt}; -use alloy_rpc_types_eth::{pubsub::TransactionReceiptsParams, Filter, Log}; +use alloy_rpc_types_eth::{pubsub::TransactionReceiptsParams, Filter}; use futures::StreamExt; use reth_chain_state::CanonStateSubscriptions; use reth_primitives_traits::TransactionMeta; @@ -17,21 +17,43 @@ pub trait EthSubscriptions: RpcNodeCore + EthApiTypes> { /// Returns a stream that yields matching logs from canonical chain updates. - fn log_stream(&self, filter: Filter) -> impl futures::Stream + Send + Unpin { - self.provider() - .canonical_state_stream() - .map(move |canon_state| canon_state.block_receipts()) - .flat_map(futures::stream::iter) - .flat_map(move |(block_receipts, removed)| { - let all_logs = logs_utils::matching_block_logs_with_tx_hashes( + fn log_stream( + &self, + filter: Filter, + ) -> impl futures::Stream> + Send + Unpin { + let converter = self.converter(); + self.provider().canonical_state_stream().flat_map(move |canon_state| { + let reverted_chains = canon_state.reverted(); + let committed_chain = canon_state.committed(); + let reverted = reverted_chains.iter().flat_map(|chain| { + chain.blocks_and_receipts().map(|(block, receipts)| (block, receipts, true)) + }); + let committed = committed_chain + .blocks_and_receipts() + .map(|(block, receipts)| (block, receipts, false)); + let mut all_logs = Vec::new(); + + for (block, receipts, removed) in reverted.chain(committed) { + let result = logs_utils::matching_block_logs_with_tx_hashes( + converter, &filter, - block_receipts.block, - block_receipts.timestamp, - block_receipts.tx_receipts.iter().map(|(tx, receipt)| (*tx, receipt)), + block.sealed_header(), + block + .transactions_recovered() + .zip(receipts.iter()) + .map(|(tx, receipt)| (*tx.tx_hash(), receipt)), removed, ); - futures::stream::iter(all_logs) - }) + match result { + Ok(logs) => all_logs.extend(logs), + Err(err) => { + error!(target = "rpc", %err, "Failed to convert logs"); + } + } + } + + futures::stream::iter(all_logs) + }) } /// Returns a stream that yields new block headers from canonical chain updates. @@ -117,7 +139,7 @@ pub trait EthSubscriptions: return None; } - match converter.convert_receipts_with_block(inputs, block.sealed_block()) { + match converter.convert_receipts(inputs, block.sealed_block()) { Ok(rpc_receipts) => Some(rpc_receipts), Err(err) => { error!(target = "rpc", %err, "Failed to convert receipts"); diff --git a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs index 80aa5bffb21..5909eb7338d 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/transaction.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/transaction.rs @@ -271,9 +271,10 @@ pub trait EthTransactions: LoadTransaction { { async move { match self.load_transaction_and_receipt(hash).await? { - Some((tx, meta, receipt, all_receipts)) => { - self.build_transaction_receipt(tx, meta, receipt, all_receipts).await.map(Some) - } + Some((tx, meta, receipt, all_receipts, block)) => self + .build_transaction_receipt(tx, meta, receipt, all_receipts, block) + .await + .map(Some), None => Ok(None), } } @@ -293,6 +294,7 @@ pub trait EthTransactions: LoadTransaction { TransactionMeta, ProviderReceipt, Option>>>, + Option>>>, )>, Self::Error, >, @@ -310,7 +312,7 @@ pub trait EthTransactions: LoadTransaction { if let Some(all_receipts) = cached.receipts.clone() && let Some(receipt) = all_receipts.get(cached.tx_index).cloned() { - return Ok(Some((tx, meta, receipt, Some(all_receipts)))); + return Ok(Some((tx, meta, receipt, Some(all_receipts), Some(cached.block)))); } // Block still cached but receipts evicted — fetch via cache since @@ -323,7 +325,7 @@ pub trait EthTransactions: LoadTransaction { .map_err(Self::Error::from_eth_err)? && let Some(receipt) = receipts.get(cached.tx_index).cloned() { - return Ok(Some((tx, meta, receipt, Some(receipts)))); + return Ok(Some((tx, meta, receipt, Some(receipts), Some(cached.block)))); } } @@ -341,7 +343,7 @@ pub trait EthTransactions: LoadTransaction { let receipt = provider.receipt_by_hash(hash).map_err(Self::Error::from_eth_err)?; - Ok(receipt.map(|receipt| (tx, meta, receipt, None))) + Ok(receipt.map(|receipt| (tx, meta, receipt, None, None))) }) .await } diff --git a/crates/rpc/rpc-eth-types/src/block.rs b/crates/rpc/rpc-eth-types/src/block.rs index f47ecc0b534..e2f5cd3f94e 100644 --- a/crates/rpc/rpc-eth-types/src/block.rs +++ b/crates/rpc/rpc-eth-types/src/block.rs @@ -183,7 +183,7 @@ where calculate_gas_used_and_next_log_index(meta.index, all_receipts); converter - .convert_receipts_with_block( + .convert_receipts( vec![ConvertReceiptInput { tx: tx.recovered_tx(), gas_used: receipt.cumulative_gas_used() - gas_used, diff --git a/crates/rpc/rpc-eth-types/src/logs_utils.rs b/crates/rpc/rpc-eth-types/src/logs_utils.rs index 8d9100c4944..495e4e15673 100644 --- a/crates/rpc/rpc-eth-types/src/logs_utils.rs +++ b/crates/rpc/rpc-eth-types/src/logs_utils.rs @@ -2,31 +2,38 @@ //! //! Log parsing for building filter. -use alloy_consensus::{transaction::TxHashRef, TxReceipt}; -use alloy_eips::BlockNumHash; +use crate::EthApiError; +use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt}; use alloy_primitives::TxHash; use alloy_rpc_types_eth::{Filter, Log}; +use jsonrpsee_types::ErrorObject; use reth_chainspec::ChainInfo; use reth_errors::ProviderError; -use reth_primitives_traits::{BlockBody, RecoveredBlock, SignedTransaction}; +use reth_primitives_traits::{ + BlockBody, NodePrimitives, RecoveredBlock, SealedHeaderFor, SignedTransaction, +}; +use reth_rpc_convert::{RpcConvert, RpcLog}; use reth_storage_api::{BlockReader, ProviderBlock}; use std::sync::Arc; use thiserror::Error; -/// Returns all matching of a block's receipts when the transaction hashes are known. -pub fn matching_block_logs_with_tx_hashes<'a, I, R>( +/// Returns all matching and converted logs of a block's receipts when the transaction hashes are +/// known. +pub fn matching_block_logs_with_tx_hashes<'a, I, R, C>( + converter: &C, filter: &Filter, - block_num_hash: BlockNumHash, - block_timestamp: u64, + header: &SealedHeaderFor, tx_hashes_and_receipts: I, removed: bool, -) -> Vec +) -> Result>, C::Error> where I: IntoIterator, R: TxReceipt + 'a, + C: RpcConvert>, { + let block_num_hash = header.num_hash(); if !filter.matches_block(&block_num_hash) { - return vec![]; + return Ok(vec![]) } let mut all_logs = Vec::new(); @@ -46,14 +53,14 @@ where transaction_index: Some(receipt_idx as u64), log_index: Some(log_index), removed, - block_timestamp: Some(block_timestamp), + block_timestamp: Some(header.timestamp()), }; - all_logs.push(log); + all_logs.push(converter.convert_log(log, receipt, header)?); } log_index += 1; } } - all_logs + Ok(all_logs) } /// Helper enum to fetch a transaction either from a block or from the provider. @@ -65,20 +72,22 @@ pub enum ProviderOrBlock<'a, P: BlockReader> { Block(Arc>>), } -/// Appends all matching logs of a block's receipts. +/// Appends all matching and converted logs of a block's receipts. /// If the log matches, look up the corresponding transaction hash. -pub fn append_matching_block_logs

( - all_logs: &mut Vec, +pub fn append_matching_block_logs( + all_logs: &mut Vec>, + converter: &C, provider_or_block: ProviderOrBlock<'_, P>, filter: &Filter, - block_num_hash: BlockNumHash, + header: &SealedHeaderFor, receipts: &[P::Receipt], removed: bool, - block_timestamp: u64, -) -> Result<(), ProviderError> +) -> Result<(), EthApiError> where P: BlockReader, + C: RpcConvert, Receipt = P::Receipt>>, { + let block_num_hash = header.num_hash(); if !filter.matches_block(&block_num_hash) { return Ok(()); } @@ -140,13 +149,17 @@ where transaction_index: Some(receipt_idx as u64), log_index: Some(log_index), removed, - block_timestamp: Some(block_timestamp), + block_timestamp: Some(header.timestamp()), }; + let log = converter + .convert_log(log, receipt, header) + .map_err(|err| EthApiError::other(Into::>::into(err)))?; all_logs.push(log); } log_index += 1; } } + Ok(()) } diff --git a/crates/rpc/rpc-eth-types/src/receipt.rs b/crates/rpc/rpc-eth-types/src/receipt.rs index 97e888d7a99..6a711853c6c 100644 --- a/crates/rpc/rpc-eth-types/src/receipt.rs +++ b/crates/rpc/rpc-eth-types/src/receipt.rs @@ -7,7 +7,7 @@ use alloy_primitives::{Address, TxKind}; use alloy_rpc_types_eth::{Log, TransactionReceipt}; use reth_chainspec::EthChainSpec; use reth_ethereum_primitives::Receipt; -use reth_primitives_traits::{NodePrimitives, TransactionMeta}; +use reth_primitives_traits::{NodePrimitives, SealedBlock, SealedHeaderFor, TransactionMeta}; use reth_rpc_convert::transaction::{ConvertReceiptInput, ReceiptConverter}; use std::sync::Arc; @@ -116,11 +116,22 @@ where Builder: Fn(N::Receipt, usize, TransactionMeta) -> Rpc + 'static, { type RpcReceipt = TransactionReceipt; + type RpcLog = Log; type Error = EthApiError; + fn convert_log( + &self, + log: Log, + _receipt: &N::Receipt, + _header: &SealedHeaderFor, + ) -> Result { + Ok(log) + } + fn convert_receipts( &self, inputs: Vec>, + _block: &SealedBlock, ) -> Result, Self::Error> { let mut receipts = Vec::with_capacity(inputs.len()); diff --git a/crates/rpc/rpc/src/engine.rs b/crates/rpc/rpc/src/engine.rs index d8409c2694b..c7ee36f0b7b 100644 --- a/crates/rpc/rpc/src/engine.rs +++ b/crates/rpc/rpc/src/engine.rs @@ -1,7 +1,7 @@ use alloy_eips::{BlockId, BlockNumberOrTag}; use alloy_primitives::{Address, Bytes, B256, U256, U64}; use alloy_rpc_types_eth::{ - state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, Log, SyncStatus, + state::StateOverride, BlockOverrides, EIP1186AccountProofResponse, Filter, SyncStatus, }; use alloy_serde::JsonStorageKey; use jsonrpsee::core::RpcResult as Result; @@ -11,7 +11,8 @@ use reth_rpc_convert::RpcTxReq; /// Re-export for convenience pub use reth_rpc_engine_api::EngineApi; use reth_rpc_eth_api::{ - EngineEthFilter, FullEthApiTypes, QueryLimits, RpcBlock, RpcHeader, RpcReceipt, RpcTransaction, + EngineEthFilter, FullEthApiTypes, QueryLimits, RpcBlock, RpcHeader, RpcLog, RpcReceipt, + RpcTransaction, }; use serde_json::Value; use tracing_futures::Instrument; @@ -43,6 +44,7 @@ impl RpcTxReq, RpcBlock, RpcReceipt, + RpcLog, > for EngineEthApi where Eth: EthApiServer< @@ -53,7 +55,7 @@ where RpcHeader, TxTy, > + FullEthApiTypes, - EthFilter: EngineEthFilter, + EthFilter: EngineEthFilter>, { /// Handler for: `eth_syncing` fn syncing(&self) -> Result { @@ -133,7 +135,7 @@ where } /// Handler for `eth_getLogs` - async fn logs(&self, filter: Filter) -> Result> { + async fn logs(&self, filter: Filter) -> Result>> { self.eth_filter.logs(filter, QueryLimits::no_limits()).instrument(engine_span!()).await } diff --git a/crates/rpc/rpc/src/eth/filter.rs b/crates/rpc/rpc/src/eth/filter.rs index e23dc4ded82..f201b2f9418 100644 --- a/crates/rpc/rpc/src/eth/filter.rs +++ b/crates/rpc/rpc/src/eth/filter.rs @@ -4,8 +4,7 @@ use alloy_consensus::BlockHeader; use alloy_eips::BlockNumberOrTag; use alloy_primitives::{Sealable, TxHash}; use alloy_rpc_types_eth::{ - BlockNumHash, Filter, FilterBlockOption, FilterChanges, FilterId, Log, - PendingTransactionFilterKind, + Filter, FilterBlockOption, FilterChanges, FilterId, PendingTransactionFilterKind, }; use async_trait::async_trait; use futures::{ @@ -20,7 +19,7 @@ use reth_primitives_traits::{NodePrimitives, SealedHeader}; use reth_rpc_eth_api::{ helpers::{EthBlocks, LoadReceipt}, EngineEthFilter, EthApiTypes, EthFilterApiServer, FullEthApiTypes, QueryLimits, RpcConvert, - RpcNodeCoreExt, RpcTransaction, + RpcLog, RpcNodeCoreExt, RpcTransaction, }; use reth_rpc_eth_types::{ logs_utils::{self, append_matching_block_logs, ProviderOrBlock}, @@ -48,7 +47,7 @@ use tokio::{ }; use tracing::{debug, error, trace}; -impl EngineEthFilter for EthFilter +impl EngineEthFilter> for EthFilter where Eth: FullEthApiTypes + RpcNodeCoreExt @@ -61,7 +60,7 @@ where &self, filter: Filter, limits: QueryLimits, - ) -> impl Future>> + Send { + ) -> impl Future>>> + Send { trace!(target: "rpc::eth", "Serving eth_getLogs"); self.logs_for_filter(filter, limits).map_err(|e| e.into()) } @@ -221,7 +220,10 @@ where pub async fn filter_changes( &self, id: FilterId, - ) -> Result>, EthFilterError> { + ) -> Result< + FilterChanges, RpcLog>, + EthFilterError, + > { let info = self.provider().chain_info()?; let best_number = info.best_number; @@ -247,7 +249,14 @@ where }; match kind { - FilterKind::PendingTransaction(filter) => Ok(filter.drain().await), + FilterKind::PendingTransaction(filter) => Ok(match filter.drain().await { + FilterChanges::Empty => FilterChanges::Empty, + FilterChanges::Hashes(hashes) => FilterChanges::Hashes(hashes), + FilterChanges::Transactions(transactions) => { + FilterChanges::Transactions(transactions) + } + FilterChanges::Logs(_) => unreachable!("pending transaction filter returned logs"), + }), FilterKind::Block => { // Note: we need to fetch the block hashes from inclusive range // [start_block..best_block] @@ -302,7 +311,10 @@ where /// Returns an error if no matching log filter exists. /// /// Handler for `eth_getFilterLogs` - pub async fn filter_logs(&self, id: FilterId) -> Result, EthFilterError> { + pub async fn filter_logs( + &self, + id: FilterId, + ) -> Result>, EthFilterError> { let filter = { let mut filters = self.inner.active_filters.inner.lock().await; let filter = @@ -324,13 +336,14 @@ where &self, filter: Filter, limits: QueryLimits, - ) -> Result, EthFilterError> { + ) -> Result>, EthFilterError> { self.inner.clone().logs_for_filter(filter, limits).await } } #[async_trait] -impl EthFilterApiServer> for EthFilter +impl EthFilterApiServer, RpcLog> + for EthFilter where Eth: FullEthApiTypes + RpcNodeCoreExt + LoadReceipt + EthBlocks + 'static, { @@ -381,7 +394,8 @@ where async fn filter_changes( &self, id: FilterId, - ) -> RpcResult>> { + ) -> RpcResult, RpcLog>> + { trace!(target: "rpc::eth", "Serving eth_getFilterChanges"); Ok(Self::filter_changes(self, id).await?) } @@ -391,7 +405,7 @@ where /// Returns an error if no matching log filter exists. /// /// Handler for `eth_getFilterLogs` - async fn filter_logs(&self, id: FilterId) -> RpcResult> { + async fn filter_logs(&self, id: FilterId) -> RpcResult>> { trace!(target: "rpc::eth", "Serving eth_getFilterLogs"); Ok(Self::filter_logs(self, id).await?) } @@ -411,7 +425,7 @@ where /// Returns logs matching given filter object. /// /// Handler for `eth_getLogs` - async fn logs(&self, filter: Filter) -> RpcResult> { + async fn logs(&self, filter: Filter) -> RpcResult>> { trace!(target: "rpc::eth", "Serving eth_getLogs"); Ok(self.logs_for_filter(filter, self.inner.query_limits).await?) } @@ -468,7 +482,7 @@ where self: Arc, filter: Filter, limits: QueryLimits, - ) -> Result, EthFilterError> { + ) -> Result>, EthFilterError> { match filter.block_option { FilterBlockOption::AtBlockHash(block_hash) => { // First try to get cached block and receipts, as it's likely they're already cached @@ -478,42 +492,38 @@ where return Err(ProviderError::HeaderNotFound(block_hash.into()).into()) }; - // Read number and timestamp from cached block or provider header - let (block_number, block_timestamp) = if let Some(block) = &maybe_block { - (block.header().number(), block.header().timestamp()) + let header = if let Some(block) = &maybe_block { + block.clone_sealed_header() } else { let header = self .provider() .header_by_hash_or_number(block_hash.into())? .ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?; - (header.number(), header.timestamp()) + SealedHeader::new(header, block_hash) }; // Check if the block has been pruned (EIP-4444) let earliest_block = self.provider().earliest_block_number()?; - if block_number < earliest_block { + if header.number() < earliest_block { return Err(EthApiError::PrunedHistoryUnavailable { - requested: block_number, + requested: header.number(), earliest_available: earliest_block, } .into()); } - let block_num_hash = BlockNumHash::new(block_number, block_hash); - let mut all_logs = Vec::new(); append_matching_block_logs( &mut all_logs, + self.eth_api.converter(), maybe_block .map(ProviderOrBlock::Block) .unwrap_or_else(|| ProviderOrBlock::Provider(self.provider())), &filter, - block_num_hash, + &header, &receipts, false, - block_timestamp, )?; - Ok(all_logs) } FilterBlockOption::Range { from_block, to_block } => { @@ -537,18 +547,17 @@ where if pending_block.block.number() > info.best_number { // only consider the pending block if it is ahead of the chain let mut all_logs = Vec::new(); - let timestamp = pending_block.block.timestamp(); - let block_num_hash = pending_block.block.num_hash(); + let header = pending_block.block.clone_sealed_header(); append_matching_block_logs( &mut all_logs, + self.eth_api.converter(), ProviderOrBlock::::Block(pending_block.block), &filter, - block_num_hash, + &header, &pending_block.receipts, false, // removed = false for pending blocks - timestamp, )?; - return Ok(all_logs); + return Ok(all_logs) } } } @@ -635,7 +644,7 @@ where from_block: u64, to_block: u64, limits: QueryLimits, - ) -> Result, EthFilterError> { + ) -> Result>, EthFilterError> { trace!(target: "rpc::eth::filter", from=from_block, to=to_block, ?filter, "finding logs in range"); // perform boundary checks first @@ -674,7 +683,7 @@ where from_block: u64, to_block: u64, limits: QueryLimits, - ) -> Result, EthFilterError> { + ) -> Result>, EthFilterError> { let mut all_logs = Vec::new(); let mut matching_headers = Vec::new(); @@ -728,14 +737,14 @@ where let num_hash = header.num_hash(); append_matching_block_logs( &mut all_logs, + self.eth_api.converter(), recovered_block .map(ProviderOrBlock::Block) .unwrap_or_else(|| ProviderOrBlock::Provider(self.provider())), filter, - num_hash, + &header, &receipts, false, - header.timestamp(), )?; // size check but only if range is multiple blocks, so we always return all diff --git a/crates/rpc/rpc/src/eth/pubsub.rs b/crates/rpc/rpc/src/eth/pubsub.rs index 3bc18055f5f..3f510ce60a2 100644 --- a/crates/rpc/rpc/src/eth/pubsub.rs +++ b/crates/rpc/rpc/src/eth/pubsub.rs @@ -7,7 +7,7 @@ use alloy_rpc_types_eth::{ pubsub::{ Params, PubSubSyncStatus, SubscriptionKind, SyncStatusMetadata, TransactionReceiptsParams, }, - Filter, Log, + Filter, }; use futures::StreamExt; use jsonrpsee::{ @@ -17,7 +17,8 @@ use reth_chain_state::CanonStateSubscriptions; use reth_network_api::NetworkInfo; use reth_rpc_convert::RpcHeader; use reth_rpc_eth_api::{ - helpers::EthSubscriptions, pubsub::EthPubSubApiServer, RpcConvert, RpcNodeCore, RpcTransaction, + helpers::EthSubscriptions, pubsub::EthPubSubApiServer, RpcConvert, RpcLog, RpcNodeCore, + RpcTransaction, }; use reth_rpc_server_types::result::{internal_rpc_err, invalid_params_rpc_err}; use reth_storage_api::BlockNumReader; @@ -76,7 +77,7 @@ where } /// Returns a stream that yields matching logs. - pub fn log_stream(&self, filter: Filter) -> impl Stream { + pub fn log_stream(&self, filter: Filter) -> impl Stream> { self.inner.eth_api.log_stream(filter) } From 22583a0e96d7b3ee30ecbc8fbf6b26f5938fd698 Mon Sep 17 00:00:00 2001 From: figtracer Date: Tue, 4 Aug 2026 19:47:24 +0100 Subject: [PATCH 091/148] feat(provider): return header with transaction metadata (#26571) Co-authored-by: Matthias Seitz --- .../src/providers/blockchain_provider.rs | 34 ++++++++++- .../provider/src/providers/consistent.rs | 25 +++++++- .../provider/src/providers/database/mod.rs | 11 +++- .../src/providers/database/provider.rs | 57 +++++++++++-------- .../storage/storage-api/src/transactions.rs | 31 +++++++++- 5 files changed, 128 insertions(+), 30 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 28729865bbc..607c910499a 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -7,7 +7,7 @@ use crate::{ BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, DatabaseProviderFactory, HashedPostStateProvider, HeaderProvider, ProviderError, - ProviderFactory, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, + ProviderFactory, ProviderHeader, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; @@ -605,6 +605,15 @@ impl TransactionsProvider for BlockchainProvider { self.consistent_provider()?.transaction_by_hash_with_meta(tx_hash) } + fn transaction_by_hash_with_meta_and_header( + &self, + tx_hash: TxHash, + ) -> ProviderResult< + Option<(Self::Transaction, TransactionMeta, SealedHeader>)>, + > { + self.consistent_provider()?.transaction_by_hash_with_meta_and_header(tx_hash) + } + fn transactions_by_block( &self, id: BlockHashOrNumber, @@ -1017,7 +1026,7 @@ mod tests { }, BlockWriter, CanonChainTracker, ProviderFactory, SaveBlocksInput, }; - use alloy_consensus::constants::EMPTY_ROOT_HASH; + use alloy_consensus::{constants::EMPTY_ROOT_HASH, transaction::TransactionMeta, BlockHeader}; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{keccak256, Address, BlockNumber, TxNumber, B256, U256}; use itertools::Itertools; @@ -2685,6 +2694,27 @@ mod tests { ), B256::random() ), + ( + ONE, + transaction_by_hash_with_meta_and_header, + |block: &SealedBlock, _: TxNumber, tx_hash: B256, _: &Vec>| ( + tx_hash, + Some(( + block.body().transactions[test_tx_index].clone(), + TransactionMeta { + tx_hash, + index: test_tx_index as u64, + block_hash: block.hash(), + block_number: block.number, + base_fee: block.base_fee_per_gas(), + excess_blob_gas: block.excess_blob_gas(), + timestamp: block.timestamp(), + }, + block.clone_sealed_header(), + )) + ), + B256::random() + ), ( ONE, block_by_transaction_id, diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index 89c9795d004..2fbd3920105 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -3,7 +3,7 @@ use crate::{ providers::{StaticFileProvider, StaticFileProviderRWRefMut}, to_range, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, ChainSpecProvider, ChangeSetReader, HeaderProvider, - ProviderError, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, + ProviderError, ProviderHeader, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; @@ -857,6 +857,29 @@ impl TransactionsProvider for ConsistentProvider { self.storage_provider.transaction_by_hash_with_meta(tx_hash) } + fn transaction_by_hash_with_meta_and_header( + &self, + tx_hash: TxHash, + ) -> ProviderResult< + Option<(Self::Transaction, TransactionMeta, SealedHeader>)>, + > { + if let Some((tx, meta, header)) = self.head_block.as_ref().and_then(|head| { + head.chain().find_map(|block_state| { + block_state.find_indexed(tx_hash).map(|indexed| { + ( + indexed.tx().clone(), + indexed.meta(), + block_state.block_ref().recovered_block().clone_sealed_header(), + ) + }) + }) + }) { + return Ok(Some((tx, meta, header))) + } + + self.storage_provider.transaction_by_hash_with_meta_and_header(tx_hash) + } + fn transactions_by_block( &self, id: BlockHashOrNumber, diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 7d6186c451c..7167ed6d45e 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -7,7 +7,7 @@ use crate::{ traits::{BlockSource, ReceiptProvider}, BalProvider, BalStoreHandle, BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, EitherWriterDestination, HashedPostStateProvider, HeaderProvider, - HeaderSyncGapProvider, InMemoryBalStore, MetadataProvider, ProviderError, + HeaderSyncGapProvider, InMemoryBalStore, MetadataProvider, ProviderError, ProviderHeader, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StaticFileProviderFactory, StaticFileWriter, TransactionVariant, TransactionsProvider, }; @@ -825,6 +825,15 @@ impl TransactionsProvider for ProviderFactory { self.provider()?.transaction_by_hash_with_meta(tx_hash) } + fn transaction_by_hash_with_meta_and_header( + &self, + tx_hash: TxHash, + ) -> ProviderResult< + Option<(Self::Transaction, TransactionMeta, SealedHeader>)>, + > { + self.provider()?.transaction_by_hash_with_meta_and_header(tx_hash) + } + fn transactions_by_block( &self, id: BlockHashOrNumber, diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 96d891e37e0..3dc41a9abdc 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -16,10 +16,10 @@ use crate::{ DBProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, HeaderProvider, HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, PersistenceFrontiers, - ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RawRocksDBBatch, RevertsInit, - RocksBatchArg, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateWriter, - StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, - TransactionsProvider, TransactionsProviderExt, TrieWriter, + ProviderError, ProviderHeader, PruneCheckpointReader, PruneCheckpointWriter, RawRocksDBBatch, + RevertsInit, RocksBatchArg, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, + StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, + TransactionVariant, TransactionsProvider, TransactionsProviderExt, TrieWriter, }; use alloy_consensus::{ transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, @@ -2107,31 +2107,40 @@ impl TransactionsProvider for Datab &self, tx_hash: TxHash, ) -> ProviderResult> { + Ok(self + .transaction_by_hash_with_meta_and_header(tx_hash)? + .map(|(transaction, meta, _)| (transaction, meta))) + } + + fn transaction_by_hash_with_meta_and_header( + &self, + tx_hash: TxHash, + ) -> ProviderResult< + Option<(Self::Transaction, TransactionMeta, SealedHeader>)>, + > { if let Some(transaction_id) = self.transaction_id(tx_hash)? && let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? && let Some(block_number) = self.block_by_transaction_id(transaction_id)? && - let Some(sealed_header) = self.sealed_header(block_number)? + let Some(sealed_header) = self.sealed_header(block_number)? && + let Some(block_body) = self.block_body_indices(block_number)? { - let (header, block_hash) = sealed_header.split(); - if let Some(block_body) = self.block_body_indices(block_number)? { - // the index of the tx in the block is the offset: - // len([start..tx_id]) - // NOTE: `transaction_id` is always `>=` the block's first - // index - let index = transaction_id - block_body.first_tx_num(); - - let meta = TransactionMeta { - tx_hash, - index, - block_hash, - block_number, - base_fee: header.base_fee_per_gas(), - excess_blob_gas: header.excess_blob_gas(), - timestamp: header.timestamp(), - }; + // the index of the tx in the block is the offset: + // len([start..tx_id]) + // NOTE: `transaction_id` is always `>=` the block's first + // index + let index = transaction_id - block_body.first_tx_num(); + + let meta = TransactionMeta { + tx_hash, + index, + block_hash: sealed_header.hash(), + block_number, + base_fee: sealed_header.base_fee_per_gas(), + excess_blob_gas: sealed_header.excess_blob_gas(), + timestamp: sealed_header.timestamp(), + }; - return Ok(Some((transaction, meta))) - } + return Ok(Some((transaction, meta, sealed_header))) } Ok(None) diff --git a/crates/storage/storage-api/src/transactions.rs b/crates/storage/storage-api/src/transactions.rs index 354504d9667..2b32465aadb 100644 --- a/crates/storage/storage-api/src/transactions.rs +++ b/crates/storage/storage-api/src/transactions.rs @@ -1,10 +1,10 @@ -use crate::{BlockNumReader, BlockReader}; +use crate::{BlockNumReader, BlockReader, HeaderProvider}; use alloc::vec::Vec; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{Address, BlockNumber, TxHash, TxNumber}; use core::ops::{Range, RangeBounds, RangeInclusive}; -use reth_primitives_traits::SignedTransaction; +use reth_primitives_traits::{SealedHeader, SignedTransaction}; use reth_storage_errors::provider::{ProviderError, ProviderResult}; /// Enum to control transaction hash inclusion. @@ -49,6 +49,33 @@ pub trait TransactionsProvider: BlockNumReader + Send { hash: TxHash, ) -> ProviderResult>; + /// Get transaction by transaction hash, additional metadata, and the sealed block header. + /// + /// Implementers that already load the header while constructing [`TransactionMeta`] should + /// override this method to return it without another lookup. + #[auto_impl(keep_default_for(&, Arc))] + #[expect(clippy::type_complexity)] + fn transaction_by_hash_with_meta_and_header( + &self, + hash: TxHash, + ) -> ProviderResult< + Option<( + Self::Transaction, + TransactionMeta, + SealedHeader<::Header>, + )>, + > + where + Self: HeaderProvider, + { + let Some((transaction, meta)) = self.transaction_by_hash_with_meta(hash)? else { + return Ok(None) + }; + let Some(header) = self.sealed_header_by_hash(meta.block_hash)? else { return Ok(None) }; + + Ok(Some((transaction, meta, header))) + } + /// Get transactions by block id. fn transactions_by_block( &self, From 92855d2643e407fa2a84c29a48c6c199a7e6ff52 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 4 Aug 2026 22:06:46 +0200 Subject: [PATCH 092/148] test(engine): manage finality in e2e reorg flows (#26584) --- .../tests/e2e-testsuite/main.rs | 20 +++++-- .../engine/tree/tests/e2e-testsuite/main.rs | 59 +++++++++++++++---- 2 files changed, 62 insertions(+), 17 deletions(-) diff --git a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs index 684719153b7..920fd160df9 100644 --- a/crates/e2e-test-utils/tests/e2e-testsuite/main.rs +++ b/crates/e2e-test-utils/tests/e2e-testsuite/main.rs @@ -8,9 +8,9 @@ use reth_e2e_test_utils::{ test_rlp_utils::{generate_test_blocks, write_blocks_to_rlp}, testsuite::{ actions::{ - Action, AssertChainTip, AssertMineBlock, CaptureBlock, CaptureBlockOnNode, - CompareNodeChainTips, CreateFork, MakeCanonical, ProduceBlocks, ReorgTo, - SelectActiveNode, UpdateBlockInfo, + Action, AssertChainTip, AssertMineBlock, BlockReference, CaptureBlock, + CaptureBlockOnNode, CompareNodeChainTips, CreateFork, FinalizeBlock, MakeCanonical, + ProduceBlocks, ReorgTo, SelectActiveNode, UpdateBlockInfo, }, setup::{NetworkSetup, Setup}, Environment, TestBuilder, @@ -250,9 +250,19 @@ async fn test_testsuite_reorg_with_tagging() -> Result<()> { let test = TestBuilder::new() .with_setup(setup) - .with_action(ProduceBlocks::::new(3)) // produce blocks 1, 2, 3 + .with_action(ProduceBlocks::::new(1)) // produce block 1 + .with_action(CaptureBlock::new("fork_base")) + .with_action(ProduceBlocks::::new(2)) // produce blocks 2, 3 + .with_action(CaptureBlock::new("main_tip")) .with_action(MakeCanonical::new()) // make main chain tip canonical - .with_action(CreateFork::::new(1, 2)) // fork from block 1, produce blocks 2', 3' + // block production finalizes the produced blocks, so re-establish finality at the fork + // base: building below the finalized block is rejected as a too deep reorg + .with_action( + FinalizeBlock::::new(BlockReference::Tag("fork_base".to_string())) + .with_head(BlockReference::Tag("main_tip".to_string())), + ) + // fork from block 1, produce blocks 2', 3' + .with_action(CreateFork::::new_from_tag("fork_base", 2)) .with_action(CaptureBlock::new("fork_tip")) // tag fork tip .with_action(ReorgTo::::new_from_tag("fork_tip")); // reorg to fork tip diff --git a/crates/engine/tree/tests/e2e-testsuite/main.rs b/crates/engine/tree/tests/e2e-testsuite/main.rs index 2ad85b8c460..e83a2fa040e 100644 --- a/crates/engine/tree/tests/e2e-testsuite/main.rs +++ b/crates/engine/tree/tests/e2e-testsuite/main.rs @@ -7,10 +7,10 @@ use eyre::Result; use reth_chainspec::{ChainSpecBuilder, MAINNET}; use reth_e2e_test_utils::testsuite::{ actions::{ - BlockReference, CaptureBlock, CompareNodeChainTips, CreateFork, ExpectFcuStatus, - MakeCanonical, ProduceBlocks, ProduceBlocksLocally, ProduceInvalidBlocks, ReorgTo, - SelectActiveNode, SendForkchoiceUpdate, SendNewPayloads, SetForkBase, UpdateBlockInfo, - ValidateCanonicalTag, WaitForSync, + AssertChainTip, BlockReference, CaptureBlock, CompareNodeChainTips, CreateFork, + ExpectFcuStatus, FinalizeBlock, MakeCanonical, ProduceBlocks, ProduceBlocksLocally, + ProduceInvalidBlocks, ReorgTo, SelectActiveNode, SendForkchoiceUpdate, SendNewPayloads, + SetForkBase, UpdateBlockInfo, ValidateCanonicalTag, WaitForSync, }, setup::{NetworkSetup, Setup}, TestBuilder, @@ -76,10 +76,19 @@ async fn test_engine_tree_fcu_reorg_with_all_blocks_e2e() -> Result<()> { let test = TestBuilder::new() .with_setup(default_engine_tree_setup()) // create a main chain with 5 blocks (blocks 0-4) - .with_action(ProduceBlocks::::new(5)) + .with_action(ProduceBlocks::::new(2)) + .with_action(CaptureBlock::new("fork_base")) + .with_action(ProduceBlocks::::new(3)) + .with_action(CaptureBlock::new("main_tip")) .with_action(MakeCanonical::new()) + // block production finalizes the produced blocks, so re-establish finality at the fork + // base: building below the finalized block is rejected as a too deep reorg + .with_action( + FinalizeBlock::::new(BlockReference::Tag("fork_base".to_string())) + .with_head(BlockReference::Tag("main_tip".to_string())), + ) // create a fork from block 2 with 3 additional blocks - .with_action(CreateFork::::new(2, 3)) + .with_action(CreateFork::::new_from_tag("fork_base", 3)) .with_action(CaptureBlock::new("fork_tip")) // perform FCU to the fork tip - this should make the fork canonical .with_action(ReorgTo::::new_from_tag("fork_tip")); @@ -113,6 +122,12 @@ async fn test_engine_tree_valid_forks_with_older_canonical_head_e2e() -> Result< // create first competing chain (chain A) from fork point with 10 blocks .with_action(CreateFork::::new_from_tag("fork_point", 10)) .with_action(CaptureBlock::new("chain_a_tip")) + // producing chain A finalized its blocks, so re-establish finality at the fork point + // before building below it again + .with_action( + FinalizeBlock::::new(BlockReference::Tag("fork_point".to_string())) + .with_head(BlockReference::Tag("chain_a_tip".to_string())), + ) // create second competing chain (chain B) from same fork point with 10 blocks .with_action(CreateFork::::new_from_tag("fork_point", 10)) .with_action(CaptureBlock::new("chain_b_tip")) @@ -149,14 +164,23 @@ async fn test_engine_tree_valid_and_invalid_forks_with_older_canonical_head_e2e( // create chain A (competing chain) - first produce valid blocks, then test invalid // scenario .with_action(ReorgTo::::new_from_tag("fork_point")) + // producing chain B finalized its blocks, so re-establish finality at the fork point + // before building below it again + .with_action( + FinalizeBlock::::new(BlockReference::Tag("fork_point".to_string())) + .with_head(BlockReference::Tag("chain_b_tip".to_string())), + ) .with_action(ProduceBlocks::::new(10)) .with_action(CaptureBlock::new("chain_a_tip")) // test that FCU to chain A tip returns VALID status (it's a valid competing chain) .with_action(ExpectFcuStatus::valid("chain_a_tip")) // attempt to produce invalid blocks (which should be rejected) .with_action(ProduceInvalidBlocks::::with_invalid_at(3, 2)) - // chain B remains the canonical chain - .with_action(ValidateCanonicalTag::new("chain_b_tip")); + // the invalid block is rejected and the chain remains on the last valid block built on + // chain A: the fork point at 6, ten chain A blocks and two valid blocks on top. Chain B + // was reorged out when chain A became canonical and its blocks were finalized. + .with_action(UpdateBlockInfo::default()) + .with_action(AssertChainTip::new(18)); test.run::().await?; @@ -319,9 +343,11 @@ async fn test_engine_tree_live_sync_transition_eventually_canonical_e2e() -> Res .with_action(CaptureBlock::new("long_chain_tip")) // Verify Node 0's canonical tip is the long chain tip .with_action(ValidateCanonicalTag::new("long_chain_tip")) - // Verify Node 1's canonical tip is still the base chain tip + // Verify Node 1's canonical tip is still behind Node 0. `ValidateCanonicalTag` cannot + // be used here: it sends an FCU to node 0, for which the base chain tip is a canonical + // ancestor below its finalized block, which is rejected as a too deep reorg. .with_action(SelectActiveNode::new(1)) - .with_action(ValidateCanonicalTag::new("base_chain_tip")) + .with_action(CompareNodeChainTips::expect_different(0, 1)) // Node 1: Send FCU pointing to Node 0's long chain tip // This should trigger Node 1 to sync the missing blocks from Node 0 .with_action(ReorgTo::::new_from_tag("long_chain_tip")) @@ -365,9 +391,18 @@ async fn test_engine_tree_fcu_reorg_with_all_blocks_v2_e2e() -> Result<()> { let test = TestBuilder::new() .with_setup(v2_engine_tree_setup()) - .with_action(ProduceBlocks::::new(5)) + .with_action(ProduceBlocks::::new(2)) + .with_action(CaptureBlock::new("fork_base")) + .with_action(ProduceBlocks::::new(3)) + .with_action(CaptureBlock::new("main_tip")) .with_action(MakeCanonical::new()) - .with_action(CreateFork::::new(2, 3)) + // block production finalizes the produced blocks, so re-establish finality at the fork + // base: building below the finalized block is rejected as a too deep reorg + .with_action( + FinalizeBlock::::new(BlockReference::Tag("fork_base".to_string())) + .with_head(BlockReference::Tag("main_tip".to_string())), + ) + .with_action(CreateFork::::new_from_tag("fork_base", 3)) .with_action(CaptureBlock::new("fork_tip")) .with_action(ReorgTo::::new_from_tag("fork_tip")); From 3b4ee78a05ec6a32edf1ce93902d562b050f6307 Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Wed, 5 Aug 2026 11:55:12 +0200 Subject: [PATCH 093/148] refactor(trie): no longer set wiped field of HashedPostState (#26524) --- crates/chain-state/src/in_memory.rs | 4 +- crates/chain-state/src/memory_overlay.rs | 52 ++++- .../execution-cache/src/cached_state.rs | 5 +- .../engine/invalid-block-hooks/src/witness.rs | 6 +- .../tree/src/tree/instrumented_state.rs | 5 +- crates/engine/tree/src/tree/mod.rs | 16 +- .../engine/tree/src/tree/payload_validator.rs | 13 +- .../tree/src/tree/state_root_strategy/mod.rs | 6 +- crates/evm/evm/src/execute.rs | 3 +- crates/revm/src/test_utils.rs | 7 +- crates/revm/src/witness.rs | 215 +++++++++++------- crates/rpc/rpc-eth-types/src/cache/db.rs | 5 +- crates/rpc/rpc/src/debug.rs | 94 +++++++- crates/rpc/rpc/src/validation.rs | 6 +- .../stages/stages/src/stages/execution/mod.rs | 50 ++-- crates/stages/stages/tests/preimage.rs | 17 +- .../src/providers/blockchain_provider.rs | 22 +- .../provider/src/providers/database/mod.rs | 18 +- .../src/providers/state/historical.rs | 124 +++++++++- .../provider/src/providers/state/latest.rs | 16 +- .../provider/src/providers/state/overlay.rs | 2 +- .../storage/provider/src/test_utils/mock.rs | 7 +- crates/storage/provider/src/traits/full.rs | 7 +- crates/storage/provider/src/writer/mod.rs | 17 +- crates/storage/rpc-provider/src/lib.rs | 14 +- crates/storage/storage-api/src/macros.rs | 2 +- crates/storage/storage-api/src/noop.rs | 7 +- crates/storage/storage-api/src/state.rs | 8 +- crates/storage/storage-overlay/src/builder.rs | 2 +- crates/storage/storage-overlay/src/manager.rs | 2 +- crates/trie/common/src/hashed_state.rs | 36 ++- crates/trie/common/src/trie_data.rs | 2 +- crates/trie/db/src/state.rs | 2 +- crates/trie/db/tests/witness.rs | 35 ++- crates/trie/parallel/src/state_root_task.rs | 2 +- crates/trie/trie/src/hashed_cursor/mod.rs | 25 +- crates/trie/trie/src/test_utils.rs | 3 +- testing/ef-tests/src/cases/blockchain_test.rs | 22 +- 38 files changed, 589 insertions(+), 290 deletions(-) diff --git a/crates/chain-state/src/in_memory.rs b/crates/chain-state/src/in_memory.rs index 41e36848e60..fb68a6a6094 100644 --- a/crates/chain-state/src/in_memory.rs +++ b/crates/chain-state/src/in_memory.rs @@ -1095,8 +1095,8 @@ mod tests { fn hashed_post_state( &self, _bundle_state: &revm::database::BundleState, - ) -> HashedPostState { - HashedPostState::default() + ) -> ProviderResult { + Ok(HashedPostState::default()) } } diff --git a/crates/chain-state/src/memory_overlay.rs b/crates/chain-state/src/memory_overlay.rs index bc36d21383c..b61cd2880da 100644 --- a/crates/chain-state/src/memory_overlay.rs +++ b/crates/chain-state/src/memory_overlay.rs @@ -1,6 +1,8 @@ use super::ExecutedBlock; use alloy_consensus::BlockHeader; -use alloy_primitives::{keccak256, Address, BlockNumber, Bytes, StorageKey, StorageValue, B256}; +use alloy_primitives::{ + keccak256, Address, BlockNumber, Bytes, StorageKey, StorageValue, B256, U256, +}; use reth_errors::ProviderResult; use reth_primitives_traits::{Account, Bytecode, NodePrimitives}; use reth_storage_api::{ @@ -209,8 +211,26 @@ impl StateProofProvider for MemoryOverlayStateProviderRef<'_, } impl HashedPostStateProvider for MemoryOverlayStateProviderRef<'_, N> { - fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { - self.historical.hashed_post_state(bundle_state) + fn hashed_post_state(&self, bundle_state: &BundleState) -> ProviderResult { + let mut hashed_state = self.historical.hashed_post_state(bundle_state)?; + + for (address, account) in bundle_state.state() { + // Accounts created in this bundle cannot have parent storage to zero. + if !account.was_destroyed() || account.original_info.is_none() { + continue + } + + let hashed_address = keccak256(address); + let Some(parent_storage) = self.trie_input().state.storages.get(&hashed_address) else { + continue + }; + let storage = &mut hashed_state.storages.entry(hashed_address).or_default().storage; + for hashed_slot in parent_storage.storage.keys() { + storage.entry(*hashed_slot).or_insert(U256::ZERO); + } + } + + Ok(hashed_state) } } @@ -284,3 +304,29 @@ impl MemoryOverlayStateProvider { // Delegates all provider impls to [`MemoryOverlayStateProviderRef`] reth_storage_api::macros::delegate_provider_impls!(MemoryOverlayStateProvider where [N: NodePrimitives]); + +#[cfg(test)] +mod tests { + use super::*; + use reth_ethereum_primitives::EthPrimitives; + use reth_storage_api::noop::NoopProvider; + use revm::database::{AccountStatus, BundleAccount}; + + #[test] + fn created_and_destroyed_account_skips_in_memory_trie_aggregation() { + let address = Address::with_last_byte(1); + let provider = MemoryOverlayStateProviderRef::::new( + Box::new(NoopProvider::default()), + Vec::new(), + ); + let mut bundle_state = BundleState::default(); + bundle_state.state.insert( + address, + BundleAccount::new(None, None, Default::default(), AccountStatus::Destroyed), + ); + + provider.hashed_post_state(&bundle_state).unwrap(); + + assert!(provider.trie_input.get().is_none()); + } +} diff --git a/crates/engine/execution-cache/src/cached_state.rs b/crates/engine/execution-cache/src/cached_state.rs index 93bf8e943bc..9be40f13b56 100644 --- a/crates/engine/execution-cache/src/cached_state.rs +++ b/crates/engine/execution-cache/src/cached_state.rs @@ -1051,7 +1051,10 @@ impl BlockHashReader for CachedStateProvider { } impl HashedPostStateProvider for CachedStateProvider { - fn hashed_post_state(&self, bundle_state: &reth_revm::db::BundleState) -> HashedPostState { + fn hashed_post_state( + &self, + bundle_state: &reth_revm::db::BundleState, + ) -> ProviderResult { self.state_provider.hashed_post_state(bundle_state) } } diff --git a/crates/engine/invalid-block-hooks/src/witness.rs b/crates/engine/invalid-block-hooks/src/witness.rs index d6e9e6646da..574b8545c29 100644 --- a/crates/engine/invalid-block-hooks/src/witness.rs +++ b/crates/engine/invalid-block-hooks/src/witness.rs @@ -121,7 +121,7 @@ fn collect_execution_data( let bundle_state = db.take_bundle(); let mut codes = BTreeMap::new(); let mut preimages = BTreeMap::new(); - let mut hashed_state = db.database.hashed_post_state(&bundle_state); + let mut hashed_state = db.database.hashed_post_state(&bundle_state)?; // Collect codes db.cache.contracts.values().chain(bundle_state.contracts.values()).for_each(|code| { @@ -141,7 +141,7 @@ fn collect_execution_data( let storage = hashed_state .storages .entry(hashed_address) - .or_insert_with(|| HashedStorage::new(account.status.was_destroyed())); + .or_insert_with(|| HashedStorage::new(false)); for (slot, value) in account_data.storage { let slot_bytes = B256::from(slot); @@ -308,7 +308,7 @@ where block_prefix: &str, ) -> eyre::Result<()> { let state_provider = self.provider.state_by_block_hash(parent_header.hash())?; - let hashed_state = state_provider.hashed_post_state(bundle_state); + let hashed_state = state_provider.hashed_post_state(bundle_state)?; let (re_executed_root, trie_output) = state_provider.state_root_with_updates(hashed_state)?; diff --git a/crates/engine/tree/src/tree/instrumented_state.rs b/crates/engine/tree/src/tree/instrumented_state.rs index c14c859f77a..cdc1f7dfb3b 100644 --- a/crates/engine/tree/src/tree/instrumented_state.rs +++ b/crates/engine/tree/src/tree/instrumented_state.rs @@ -297,7 +297,10 @@ impl BlockHashReader for InstrumentedStateProvider { } impl HashedPostStateProvider for InstrumentedStateProvider { - fn hashed_post_state(&self, bundle_state: &reth_revm::db::BundleState) -> HashedPostState { + fn hashed_post_state( + &self, + bundle_state: &reth_revm::db::BundleState, + ) -> ProviderResult { self.state_provider.hashed_post_state(bundle_state) } } diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 34a85de3e9b..4b6ee75aa3b 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -30,9 +30,9 @@ use reth_primitives_traits::{ }; use reth_provider::{ BalProvider, BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader, - DatabaseProviderFactory, HashedPostStateProvider, ProviderError, SaveBlocksInput, - StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, - StorageChangeSetReader, StorageSettingsCache, TransactionVariant, + DatabaseProviderFactory, ProviderError, SaveBlocksInput, StageCheckpointReader, + StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, + StorageSettingsCache, TransactionVariant, }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; @@ -409,7 +409,6 @@ where + BlockReader + StateProviderFactory + StateReader - + HashedPostStateProvider + BalProvider + Clone + 'static, @@ -2368,7 +2367,14 @@ where .provider .get_state(block.header().number())? .ok_or_else(|| ProviderError::StateForNumberNotFound(block.header().number()))?; - let hashed_state = self.provider.hashed_post_state(execution_output.state()); + let bundle_state = execution_output.state(); + // `get_state` can return an in-memory execution outcome that retains destruction statuses. + // Hashing it requires the parent provider to expand a pre-existing destroyed account's + // storage into zero-valued slots. + let hashed_state = self + .provider + .state_by_block_hash(block.parent_hash())? + .hashed_post_state(bundle_state)?; debug!( target: "engine::tree", diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 3e4ac98f685..28b6e29e189 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -147,15 +147,15 @@ use reth_primitives_traits::{ }; use reth_provider::{ providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, - ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderBox, + ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, ProviderError, + PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_storage_overlay::OverlayManager; use reth_trie::{ hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, - LazyTrieData, + HashedPostState, KeccakKeyHasher, LazyTrieData, }; use std::{ sync::{ @@ -314,7 +314,6 @@ where + BlockNumReader + StateProviderFactory + StateReader - + HashedPostStateProvider + Clone + Send + Sync @@ -750,7 +749,6 @@ where // block conversion and receipt root computation. This is a pure CPU-bound task // (keccak256 hashing of all changed addresses and storage slots). let hashed_state_output = output.clone(); - let hashed_state_provider = self.provider.clone(); let mut hashed_state_rx = state_root_job.take_hashed_state_rx(); let mut hashed_state: LazyHashedPostState = self.runtime.spawn_blocking_named("hash-post-state", move || { @@ -762,7 +760,9 @@ where if let Some(Ok(state)) = hashed_state_rx.as_mut().map(|rx| rx.recv()) { state } else { - Arc::new(hashed_state_provider.hashed_post_state(&hashed_state_output.state)) + Arc::new(HashedPostState::from_bundle_state::( + hashed_state_output.state.state(), + )) } }); @@ -1800,7 +1800,6 @@ where + StateReader + ChangeSetReader + BlockNumReader - + HashedPostStateProvider + Clone + 'static, OverlayStateProviderFactory: DatabaseProviderROFactory diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 26422dc9666..b2d84453352 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -1041,7 +1041,7 @@ where let (fallback_tx, fallback_rx) = mpsc::channel(); executor.spawn_blocking_named("serial-root", move || { let result = (|| { - let hashed_state = Arc::new(provider.hashed_post_state(&output.state)); + let hashed_state = Arc::new(provider.hashed_post_state(&output.state)?); let (root, updates) = provider.state_root_with_updates(hashed_state.as_ref().clone())?; Ok((root, updates, hashed_state)) @@ -1061,7 +1061,7 @@ where output: &BlockExecutionOutput, ) -> ProviderResult { let provider = self.provider_builder.clone().build()?; - let hashed_state = Arc::new(provider.hashed_post_state(&output.state)); + let hashed_state = Arc::new(provider.hashed_post_state(&output.state)?); let (state_root, trie_updates) = provider.state_root_with_updates(hashed_state.as_ref().clone())?; self.metrics.state_root_task_fallback_success_total.increment(1); @@ -1239,7 +1239,7 @@ where debug!(target: "engine::tree::state_root_strategy", "Comparing trie updates with serial computation"); match state_provider_builder.build().and_then(|provider| { - let hashed_state = provider.hashed_post_state(&output.state); + let hashed_state = provider.hashed_post_state(&output.state)?; provider.state_root_with_updates(hashed_state) }) { Ok((serial_root, serial_trie_updates)) => { diff --git a/crates/evm/evm/src/execute.rs b/crates/evm/evm/src/execute.rs index d641bbcffd2..8035ac80964 100644 --- a/crates/evm/evm/src/execute.rs +++ b/crates/evm/evm/src/execute.rs @@ -514,7 +514,8 @@ where let block_access_list_hash = block_access_list.as_ref().map(|bal| compute_block_access_list_hash(bal.as_slice())); - let hashed_state = state.hashed_post_state(&db.bundle_state); + let hashed_state = + state.hashed_post_state(&db.bundle_state).map_err(BlockExecutionError::other)?; let (state_root, trie_updates) = match state_root_precomputed { Some(precomputed) => precomputed, None => state diff --git a/crates/revm/src/test_utils.rs b/crates/revm/src/test_utils.rs index abd861e4414..de906dd4ac0 100644 --- a/crates/revm/src/test_utils.rs +++ b/crates/revm/src/test_utils.rs @@ -152,8 +152,11 @@ impl StateProofProvider for StateProviderTest { } impl HashedPostStateProvider for StateProviderTest { - fn hashed_post_state(&self, bundle_state: &revm::database::BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::(bundle_state.state()) + fn hashed_post_state( + &self, + bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + Ok(HashedPostState::from_bundle_state::(bundle_state.state())) } } diff --git a/crates/revm/src/witness.rs b/crates/revm/src/witness.rs index 8503ed3f52a..3be8bad69e8 100644 --- a/crates/revm/src/witness.rs +++ b/crates/revm/src/witness.rs @@ -3,37 +3,43 @@ use alloy_primitives::{keccak256, Bytes, B256}; use reth_trie::{ExecutionWitnessMode, HashedPostState, HashedStorage}; use revm::database::State; -/// Tracks state changes during execution. -#[derive(Debug, Clone, Default)] -pub struct ExecutionWitnessRecord { - /// Records all state changes - pub hashed_state: HashedPostState, - /// Map of all contract codes (created / accessed) to their preimages that were required during - /// the execution of the block, including during state root recomputation. - /// - /// `keccak(bytecodes) => bytecodes` - pub codes: Vec, - /// Map of all hashed account and storage keys (addresses and slots) to their preimages - /// (unhashed account addresses and storage slots, respectively) that were required during - /// the execution of the block. - /// - /// `keccak(address|slot) => address|slot` - pub keys: Vec, - /// The lowest block number referenced by any BLOCKHASH opcode call during transaction - /// execution. - /// - /// This helps determine which ancestor block headers must be included in the - /// `ExecutionWitness`. - /// - /// `None` - when the BLOCKHASH opcode was not called during execution - pub lowest_block_number: Option, +/// Borrows finalized execution state for witness generation. +#[derive(Debug, Clone, Copy)] +pub struct ExecutionWitnessRecord<'a, DB> { + /// State after execution. + state: &'a State, } -impl ExecutionWitnessRecord { - /// Records the state after execution using the given witness generation mode. - pub fn record_executed_state(&mut self, statedb: &State, mode: ExecutionWitnessMode) { - self.codes = match mode { - ExecutionWitnessMode::Legacy => statedb +impl<'a, DB> ExecutionWitnessRecord<'a, DB> { + /// Creates a new record from the state after execution. + pub const fn new(state: &'a State) -> Self { + Self { state } + } + + /// Converts this record into a complete [`alloy_rpc_types_debug::ExecutionWitness`] by + /// generating state proofs and fetching ancestor block headers. + /// + /// The `block_number` is the number of the block being witnessed. Ancestor headers are + /// included based on the lowest block number referenced by BLOCKHASH opcodes during + /// execution, or just the parent header if BLOCKHASH was not called. + #[cfg(feature = "witness")] + pub fn into_execution_witness( + self, + state_provider: &SP, + headers_provider: &HP, + block_number: u64, + mode: ExecutionWitnessMode, + ) -> reth_storage_errors::provider::ProviderResult + where + SP: reth_storage_api::HashedPostStateProvider + + reth_storage_api::StateProofProvider + + ?Sized, + HP: reth_storage_api::HeaderProvider + ?Sized, + HP::Header: alloy_rlp::Encodable, + { + let codes = match mode { + ExecutionWitnessMode::Legacy => self + .state .cache .contracts .values() @@ -43,11 +49,12 @@ impl ExecutionWitnessRecord { // a contract is created within the block // the contract only exists in bundle state, therefore we need // to include them as well - statedb.bundle_state.contracts.values().map(|code| code.original_bytes()), + self.state.bundle_state.contracts.values().map(|code| code.original_bytes()), ) .collect(), ExecutionWitnessMode::Canonical => { - let mut codes: Vec<_> = statedb + let mut codes: Vec<_> = self + .state .cache .contracts .values() @@ -59,79 +66,125 @@ impl ExecutionWitnessRecord { } }; - for (address, account) in &statedb.cache.accounts { + let (hashed_state, keys) = self.hashed_post_state(state_provider)?; + + let state = state_provider.witness(Default::default(), hashed_state, mode)?; + let mut exec_witness = + alloy_rpc_types_debug::ExecutionWitness { state, codes, keys, ..Default::default() }; + + let lowest_block_number = + self.state.block_hashes.lowest().map(|(block_number, _)| block_number); + let smallest = lowest_block_number.unwrap_or_else(|| block_number.saturating_sub(1)); + let range = smallest..block_number; + + exec_witness.headers = headers_provider + .headers_range(range)? + .into_iter() + .map(|header| { + let mut buf = Vec::new(); + alloy_rlp::Encodable::encode(&header, &mut buf); + buf.into() + }) + .collect(); + + Ok(exec_witness) + } + + #[cfg(feature = "witness")] + fn hashed_post_state( + &self, + state_provider: &SP, + ) -> reth_storage_errors::provider::ProviderResult<(HashedPostState, Vec)> + where + SP: reth_storage_api::HashedPostStateProvider + ?Sized, + { + let mut hashed_state = HashedPostState::default(); + let mut keys = Vec::new(); + for (address, account) in &self.state.cache.accounts { let hashed_address = keccak256(address); - self.hashed_state + hashed_state .accounts .insert(hashed_address, account.account.as_ref().map(|a| (&a.info).into())); - let storage = self - .hashed_state + let storage = hashed_state .storages .entry(hashed_address) - .or_insert_with(|| HashedStorage::new(account.status.was_destroyed())); + .or_insert_with(|| HashedStorage::new(false)); if let Some(account) = &account.account { - self.keys.push(address.to_vec().into()); + keys.push(address.to_vec().into()); for (slot, value) in &account.storage { let slot = B256::from(*slot); let hashed_slot = keccak256(slot); storage.storage.insert(hashed_slot, *value); - self.keys.push(slot.into()); + keys.push(slot.into()); } } } - self.lowest_block_number = - statedb.block_hashes.lowest().map(|(block_number, _)| block_number) - } - /// Creates the record from the state after execution. - pub fn from_executed_state(state: &State, mode: ExecutionWitnessMode) -> Self { - let mut record = Self::default(); - record.record_executed_state(state, mode); - record + // The execution cache does not contain untouched slots of a destroyed account. The + // provider expands them into explicit zero writes from the parent state; extending it last + // also ensures the bundle's final values override those collected from the cache. + hashed_state.extend(state_provider.hashed_post_state(&self.state.bundle_state)?); + Ok((hashed_state, keys)) } +} - /// Converts this record into a complete [`alloy_rpc_types_debug::ExecutionWitness`] by - /// generating state proofs and fetching ancestor block headers. - /// - /// The `block_number` is the number of the block being witnessed. Ancestor headers are - /// included based on the lowest block number referenced by BLOCKHASH opcodes during - /// execution, or just the parent header if BLOCKHASH was not called. - #[cfg(feature = "witness")] - pub fn into_execution_witness( - self, - state_provider: &SP, - headers_provider: &HP, - block_number: u64, - mode: ExecutionWitnessMode, - ) -> reth_storage_errors::provider::ProviderResult - where - SP: reth_storage_api::StateProofProvider + ?Sized, - HP: reth_storage_api::HeaderProvider + ?Sized, - HP::Header: alloy_rlp::Encodable, - { - let Self { hashed_state, codes, keys, lowest_block_number } = self; +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{Address, U256}; + use reth_storage_api::HashedPostStateProvider; + use reth_storage_errors::provider::ProviderResult; + use revm::{ + database::{states::CacheAccount, AccountStatus, BundleAccount, EmptyDB}, + state::AccountInfo, + }; - let state = state_provider.witness(Default::default(), hashed_state, mode)?; - let mut exec_witness = - alloy_rpc_types_debug::ExecutionWitness { state, codes, keys, ..Default::default() }; + #[derive(Debug)] + struct ExpandedStateProvider(HashedPostState); - let smallest = lowest_block_number.unwrap_or_else(|| block_number.saturating_sub(1)); - let range = smallest..block_number; + impl HashedPostStateProvider for ExpandedStateProvider { + fn hashed_post_state( + &self, + bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + assert!(bundle_state.state.values().any(BundleAccount::was_destroyed)); + Ok(self.0.clone()) + } + } - exec_witness.headers = headers_provider - .headers_range(range)? - .into_iter() - .map(|header| { - let mut buf = Vec::new(); - alloy_rlp::Encodable::encode(&header, &mut buf); - buf.into() - }) - .collect(); + #[test] + fn destroyed_account_storage_is_zero_expanded_without_wipe() { + let address = Address::with_last_byte(1); + let hashed_address = keccak256(address); + let hashed_slot = B256::with_last_byte(2); - Ok(exec_witness) + let mut state = State::builder().with_database(EmptyDB::default()).build(); + state.cache.accounts.insert(address, CacheAccount::new_destroyed()); + state.bundle_state.state.insert( + address, + BundleAccount::new( + Some(AccountInfo::default()), + None, + Default::default(), + AccountStatus::Destroyed, + ), + ); + + let provider = ExpandedStateProvider( + HashedPostState::default().with_accounts([(hashed_address, None)]).with_storages([( + hashed_address, + HashedStorage::from_iter([(hashed_slot, U256::ZERO)]), + )]), + ); + + let (hashed_state, _) = + ExecutionWitnessRecord::new(&state).hashed_post_state(&provider).unwrap(); + let storage = hashed_state.storages.get(&hashed_address).unwrap(); + assert!(!storage.wiped); + assert_eq!(storage.storage.get(&hashed_slot), Some(&U256::ZERO)); } } diff --git a/crates/rpc/rpc-eth-types/src/cache/db.rs b/crates/rpc/rpc-eth-types/src/cache/db.rs index 167320dd823..644b1114a0b 100644 --- a/crates/rpc/rpc-eth-types/src/cache/db.rs +++ b/crates/rpc/rpc-eth-types/src/cache/db.rs @@ -141,7 +141,10 @@ impl reth_storage_api::BlockHashReader for StateProviderTraitObjWrapper { } impl HashedPostStateProvider for StateProviderTraitObjWrapper { - fn hashed_post_state(&self, bundle_state: &BundleState) -> reth_trie::HashedPostState { + fn hashed_post_state( + &self, + bundle_state: &BundleState, + ) -> ProviderResult { self.0.hashed_post_state(bundle_state) } } diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index cf749cfd20f..24bc871691d 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -40,7 +40,8 @@ use reth_storage_api::{ use reth_tasks::{pool::BlockingTaskGuard, Runtime}; use reth_transaction_pool::TransactionPool; use reth_trie_common::{ - updates::TrieUpdates, ExecutionWitnessMode, HashedPostState, HashedStorage, + root::storage_root_unsorted, updates::TrieUpdates, ExecutionWitnessMode, HashedPostState, + HashedStorage, }; use revm::{database::states::bundle_state::BundleRetention, Database, DatabaseCommit}; use revm_inspectors::tracing::{DebugInspector, TransactionContext}; @@ -548,16 +549,21 @@ where .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { let block_executor = eth_api.evm_config().executor(&mut db); - let mut witness_record = ExecutionWitnessRecord::default(); - + let mut witness = None; let _ = block_executor .execute_with_state_closure(&block, |statedb: &State<_>| { - witness_record.record_executed_state(statedb, mode); + witness = + Some(ExecutionWitnessRecord::new(statedb).into_execution_witness( + &statedb.database.database.0, + eth_api.provider(), + block_number, + mode, + )); }) .map_err(|err| EthApiError::Internal(err.into()))?; - Ok(witness_record - .into_execution_witness(&db.database.0, eth_api.provider(), block_number, mode) + Ok(witness + .expect("state closure is called after successful execution") .map_err(EthApiError::from)?) }) .await @@ -641,18 +647,25 @@ where let balance = account.balance; let nonce = account.nonce; let code_hash = account.code_hash; - let hashed_storage = db + let (hashed_storage, status) = db .cache .accounts .get(&address) .and_then(|account| { account.account.as_ref().map(|plain_account| { - HashedStorage::from_plain_storage(account.status, plain_account.storage.iter()) + (HashedStorage::from_plain_storage(&plain_account.storage), account.status) }) }) .unwrap_or_default(); - let storage_root = - db.database.storage_root(address, hashed_storage).map_err(Eth::Error::from_eth_err)?; + let storage_root = if status.was_destroyed() { + // Destruction makes every slot not present in the cache zero, so the cache contains the + // complete storage trie for the account's new incarnation. + storage_root_unsorted( + hashed_storage.storage.into_iter().filter(|(_, value)| !value.is_zero()), + ) + } else { + db.database.storage_root(address, hashed_storage).map_err(Eth::Error::from_eth_err)? + }; Ok(Some(Account { balance, nonce, code_hash, storage_root })) } @@ -736,7 +749,10 @@ where // Merge transitions into cumulative bundle_state db.merge_transitions(BundleRetention::PlainState); // Compute state root from the accumulated state changes - let hashed_state = db.database.hashed_post_state(&db.bundle_state); + let hashed_state = db + .database + .hashed_post_state(&db.bundle_state) + .map_err(Eth::Error::from_eth_err)?; let root = db.database.state_root(hashed_state).map_err(Eth::Error::from_eth_err)?; roots.push(root); @@ -1274,3 +1290,59 @@ impl Default for BadBlockStore { Self::new(64) } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::{keccak256, U256}; + use reth_db_api::{tables, transaction::DbTxMut}; + use reth_primitives_traits::StorageEntry; + use reth_provider::test_utils::create_test_provider_factory; + use revm::{ + database::{states::StorageSlot, AccountStatus, BundleAccount, BundleState}, + state::AccountInfo as RevmAccountInfo, + }; + + #[test] + fn hashed_post_state_zeroes_destroyed_account_parent_storage() { + let factory = create_test_provider_factory(); + let address = Address::with_last_byte(1); + let old_slot = U256::from(1); + let new_slot = U256::from(2); + let old_value = U256::from(10); + let new_value = U256::from(20); + let hashed_address = keccak256(address); + let hashed_old_slot = keccak256(B256::from(old_slot)); + let hashed_new_slot = keccak256(B256::from(new_slot)); + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw + .tx_ref() + .put::( + hashed_address, + StorageEntry { key: hashed_old_slot, value: old_value }, + ) + .unwrap(); + provider_rw.commit().unwrap(); + + let mut bundle_state = BundleState::default(); + bundle_state.state.insert( + address, + BundleAccount::new( + Some(RevmAccountInfo::default()), + Some(RevmAccountInfo::default()), + std::iter::once((new_slot, StorageSlot::new_changed(U256::ZERO, new_value))) + .collect(), + AccountStatus::DestroyedChanged, + ), + ); + + let provider = factory.latest().unwrap(); + let hashed_state = provider.hashed_post_state(&bundle_state).unwrap(); + let storage = &hashed_state.storages[&hashed_address]; + + assert!(!storage.wiped); + assert_eq!(storage.storage[&hashed_old_slot], U256::ZERO); + assert_eq!(storage.storage[&hashed_new_slot], new_value); + } +} diff --git a/crates/rpc/rpc/src/validation.rs b/crates/rpc/rpc/src/validation.rs index e127a4d9028..33751dec024 100644 --- a/crates/rpc/rpc/src/validation.rs +++ b/crates/rpc/rpc/src/validation.rs @@ -36,7 +36,7 @@ use reth_primitives_traits::{ use reth_revm::{cached::CachedReads, database::StateProviderDatabase}; use reth_rpc_api::BlockSubmissionValidationApiServer; use reth_rpc_server_types::result::{internal_rpc_err, invalid_params_rpc_err}; -use reth_storage_api::{BlockReaderIdExt, StateProviderFactory}; +use reth_storage_api::{BlockReaderIdExt, HashedPostStateProvider, StateProviderFactory}; use reth_tasks::Runtime; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -235,8 +235,8 @@ where self.ensure_payment(&block, &output, &message)?; - let state_root = - state_provider.state_root(state_provider.hashed_post_state(&output.state))?; + let hashed_state = state_provider.hashed_post_state(&output.state)?; + let state_root = state_provider.state_root(hashed_state)?; if state_root != block.header().state_root() { return Err(ConsensusError::BodyStateRootDiff( diff --git a/crates/stages/stages/src/stages/execution/mod.rs b/crates/stages/stages/src/stages/execution/mod.rs index 1ba0e71fd87..61253135c85 100644 --- a/crates/stages/stages/src/stages/execution/mod.rs +++ b/crates/stages/stages/src/stages/execution/mod.rs @@ -12,9 +12,10 @@ use reth_exex::{ExExManagerHandle, ExExNotification, ExExNotificationSource}; use reth_primitives_traits::{format_gas_throughput, BlockBody, NodePrimitives}; use reth_provider::{ providers::{StaticFileProvider, StaticFileWriter}, - BlockHashReader, BlockReader, DBProvider, EitherWriter, ExecutionOutcome, HeaderProvider, - LatestStateProviderRef, OriginalValuesKnown, ProviderError, StateWriteConfig, StateWriter, - StaticFileProviderFactory, StatsReader, StoragePath, StorageSettingsCache, TransactionVariant, + BlockHashReader, BlockReader, DBProvider, EitherWriter, ExecutionOutcome, + HashedPostStateProvider, HeaderProvider, LatestStateProviderRef, OriginalValuesKnown, + ProviderError, StateWriteConfig, StateWriter, StaticFileProviderFactory, StatsReader, + StoragePath, StorageSettingsCache, TransactionVariant, }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::{ @@ -23,8 +24,6 @@ use reth_stages_api::{ UnwindInput, UnwindOutput, }; use reth_static_file_types::StaticFileSegment; -use reth_trie::{hashed_cursor::zero_destroyed_account_storage, KeccakKeyHasher}; -use reth_trie_db::DatabaseHashedCursorFactory; use std::{ cmp::{max, Ordering}, collections::BTreeMap, @@ -498,12 +497,8 @@ where provider.write_state(&state, OriginalValuesKnown::Yes, StateWriteConfig::default())?; if provider.cached_storage_settings().use_hashed_state() { - let mut hashed_state = state.hash_state_slow::(); - zero_destroyed_account_storage( - &DatabaseHashedCursorFactory::new(provider.tx_ref()), - state.bundle.state(), - &mut hashed_state, - )?; + let hashed_state = + LatestStateProviderRef::new(provider).hashed_post_state(&state.bundle)?; provider.write_hashed_state(&hashed_state.into_sorted())?; } @@ -777,7 +772,6 @@ mod tests { use reth_revm::revm::database::{AccountStatus, BundleAccount}; use reth_stages_api::StageUnitCheckpoint; use reth_testing_utils::generators; - use reth_trie::HashedPostState; use std::collections::BTreeMap; fn stage() -> ExecutionStage { @@ -806,22 +800,21 @@ mod tests { let provider = factory.database_provider_rw().unwrap(); let address = Address::repeat_byte(0x11); let hashed_address = keccak256(address); - let retained_slot = B256::repeat_byte(0x22); - let deleted_slot = B256::repeat_byte(0x33); - let retained_value = U256::from(1); + let first_slot = B256::repeat_byte(0x22); + let second_slot = B256::repeat_byte(0x33); provider .tx_ref() .put::( hashed_address, - StorageEntry { key: retained_slot, value: U256::from(2) }, + StorageEntry { key: first_slot, value: U256::from(2) }, ) .unwrap(); provider .tx_ref() .put::( hashed_address, - StorageEntry { key: deleted_slot, value: U256::from(3) }, + StorageEntry { key: second_slot, value: U256::from(3) }, ) .unwrap(); @@ -830,31 +823,18 @@ mod tests { address, BundleAccount::new( Some(Default::default()), - Some(Default::default()), + None, Default::default(), - AccountStatus::DestroyedChanged, + AccountStatus::Destroyed, ), ); - let mut hashed_state = HashedPostState::default(); - hashed_state - .storages - .entry(hashed_address) - .or_default() - .storage - .insert(retained_slot, retained_value); - - zero_destroyed_account_storage( - &DatabaseHashedCursorFactory::new(provider.tx_ref()), - state.bundle.state(), - &mut hashed_state, - ) - .unwrap(); + let hashed_state = provider.latest().hashed_post_state(&state.bundle).unwrap(); let storage = &hashed_state.storages[&hashed_address]; assert!(!storage.wiped); - assert_eq!(storage.storage[&retained_slot], retained_value); - assert_eq!(storage.storage[&deleted_slot], U256::ZERO); + assert_eq!(storage.storage[&first_slot], U256::ZERO); + assert_eq!(storage.storage[&second_slot], U256::ZERO); assert!(state.bundle.reverts.is_empty()); } diff --git a/crates/stages/stages/tests/preimage.rs b/crates/stages/stages/tests/preimage.rs index ddcb1f74808..001f160e9aa 100644 --- a/crates/stages/stages/tests/preimage.rs +++ b/crates/stages/stages/tests/preimage.rs @@ -37,7 +37,8 @@ use reth_primitives_traits::{ }; use reth_provider::{ test_utils::create_test_provider_factory_with_chain_spec, BlockNumReader, DBProvider, - DatabaseProviderFactory, HeaderProvider, OriginalValuesKnown, StateWriter, StoragePath, + DatabaseProviderFactory, HashedPostStateProvider, HeaderProvider, OriginalValuesKnown, + StateWriter, StoragePath, }; use reth_prune_types::PruneModes; use reth_revm::database::StateProviderDatabase; @@ -49,10 +50,8 @@ use reth_stages_api::{Pipeline, StageSet}; use reth_static_file::StaticFileProducer; use reth_storage_api::{StorageChangeSetReader, StorageSettings, StorageSettingsCache}; use reth_testing_utils::generators::{self, generate_key, sign_tx_with_key_pair}; -use reth_trie::{ - hashed_cursor::zero_destroyed_account_storage, HashedPostState, KeccakKeyHasher, StateRoot, -}; -use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; +use reth_trie::{HashedPostState, KeccakKeyHasher, StateRoot}; +use reth_trie_db::DatabaseStateRoot; use std::{collections::BTreeMap, path::Path, sync::Arc}; use tokio::sync::watch; @@ -1101,13 +1100,7 @@ fn execute_and_commit_block( }; let gas_used = output.gas_used; - let mut hashed_state = - HashedPostState::from_bundle_state::(output.state.state()); - zero_destroyed_account_storage( - &DatabaseHashedCursorFactory::new(provider.tx_ref()), - output.state.state(), - &mut hashed_state, - )?; + let hashed_state = provider.latest().hashed_post_state(&output.state)?; type TestStateRoot<'a, TX, A> = StateRoot< reth_trie_db::DatabaseTrieCursorFactory<&'a TX, A>, reth_trie_db::DatabaseHashedCursorFactory<&'a TX>, diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 607c910499a..c31b47cbb38 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -6,10 +6,10 @@ use crate::{ AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, - DatabaseProviderFactory, HashedPostStateProvider, HeaderProvider, ProviderError, - ProviderFactory, ProviderHeader, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, - RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateProviderFactory, - StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, + DatabaseProviderFactory, HeaderProvider, ProviderError, ProviderFactory, ProviderHeader, + PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RocksDBProviderFactory, + StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, + StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; @@ -39,10 +39,8 @@ use reth_trie::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, metrics::TrieRootMetrics, proof::{Proof, StorageProof}, - HashedPostState, KeccakKeyHasher, MultiProofTargets, StorageRoot, TrieInput, TrieInputSorted, - TrieType, + MultiProofTargets, StorageRoot, TrieInput, TrieInputSorted, TrieType, }; -use revm::database::BundleState; use std::{ ops::{RangeBounds, RangeInclusive}, sync::Arc, @@ -847,12 +845,6 @@ impl StateProviderFactory for BlockchainProvider { } } -impl HashedPostStateProvider for BlockchainProvider { - fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::(bundle_state.state()) - } -} - impl CanonChainTracker for BlockchainProvider { type Header = HeaderTy; @@ -3263,9 +3255,7 @@ mod tests { let mut state_b = HashedPostState::default(); state_b.accounts.insert(hashed_address, Some(account_b)); - state_b - .storages - .insert(hashed_address, HashedStorage::from_iter(false, [(hashed_slot, value_b)])); + state_b.storages.insert(hashed_address, HashedStorage::from_iter([(hashed_slot, value_b)])); let state_b_root = factory.latest()?.state_root(state_b.clone())?; let mut later_block = random_block( diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index 7167ed6d45e..fcb6b9ec069 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -6,10 +6,10 @@ use crate::{ to_range, traits::{BlockSource, ReceiptProvider}, BalProvider, BalStoreHandle, BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, - DatabaseProviderFactory, EitherWriterDestination, HashedPostStateProvider, HeaderProvider, - HeaderSyncGapProvider, InMemoryBalStore, MetadataProvider, ProviderError, ProviderHeader, - PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, - StaticFileProviderFactory, StaticFileWriter, TransactionVariant, TransactionsProvider, + DatabaseProviderFactory, EitherWriterDestination, HeaderProvider, HeaderSyncGapProvider, + InMemoryBalStore, MetadataProvider, ProviderError, ProviderHeader, PruneCheckpointReader, + RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StaticFileProviderFactory, + StaticFileWriter, TransactionVariant, TransactionsProvider, }; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::BlockHashOrNumber; @@ -34,8 +34,6 @@ use reth_storage_api::{ }; use reth_storage_errors::provider::ProviderResult; use reth_storage_overlay::OverlayManager; -use reth_trie::HashedPostState; -use revm::database::BundleState; use std::{ ops::{RangeBounds, RangeInclusive}, path::Path, @@ -53,8 +51,6 @@ mod save_blocks; pub use save_blocks::SaveBlocksInput; use super::ProviderNodeTypes; -use reth_trie::KeccakKeyHasher; - mod builder; pub use builder::{ProviderFactoryBuilder, ReadOnlyConfig}; @@ -969,12 +965,6 @@ impl PruneCheckpointReader for ProviderFactory { } } -impl HashedPostStateProvider for ProviderFactory { - fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::(bundle_state.state()) - } -} - impl MetadataProvider for ProviderFactory { fn get_metadata(&self, key: &str) -> ProviderResult>> { self.provider()?.get_metadata(key) diff --git a/crates/storage/provider/src/providers/state/historical.rs b/crates/storage/provider/src/providers/state/historical.rs index 8d36d979479..348e295497c 100644 --- a/crates/storage/provider/src/providers/state/historical.rs +++ b/crates/storage/provider/src/providers/state/historical.rs @@ -20,7 +20,7 @@ use reth_storage_api::{ use reth_storage_errors::provider::ProviderResult; use reth_storage_overlay::{Overlay, OverlayManager}; use reth_trie::{ - hashed_cursor::HashedPostStateCursorFactory, + hashed_cursor::{zero_destroyed_account_storage, HashedPostStateCursorFactory}, proof::{Proof, StorageProof}, trie_cursor::InMemoryTrieCursorFactory, updates::TrieUpdates, @@ -631,11 +631,41 @@ where impl HashedPostStateProvider for HistoricalStateProviderRef<'_, Provider, N> where - Provider: NodePrimitivesProvider, + Provider: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + BlockNumReader + + BlockHashReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + NodePrimitivesProvider, N: NodePrimitives, { - fn hashed_post_state(&self, bundle_state: &revm::database::BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::(bundle_state.state()) + fn hashed_post_state( + &self, + bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + let mut hashed_state = + HashedPostState::from_bundle_state::(bundle_state.state()); + if !bundle_state + .state() + .values() + .any(|account| account.was_destroyed() && account.original_info.is_some()) + { + return Ok(hashed_state) + } + + let historical = self.build_overlay(TrieInputSorted::default())?.state; + zero_destroyed_account_storage( + &HashedPostStateCursorFactory::new( + reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()), + historical.as_ref(), + ), + bundle_state.state(), + &mut hashed_state, + )?; + Ok(hashed_state) } } @@ -1446,6 +1476,92 @@ mod tests { )); } + #[test] + fn destroyed_storage_zeros_use_historical_state() { + use crate::BlockWriter; + use alloy_primitives::{keccak256, map::HashMap}; + use reth_execution_types::ExecutionOutcome; + use reth_stages_types::{StageCheckpoint, StageId}; + use reth_storage_api::{HashedPostStateProvider, StageCheckpointWriter}; + use reth_testing_utils::generators::{self, random_block_range, BlockRangeParams}; + use revm::{ + database::{AccountStatus, BundleAccount, BundleState}, + state::AccountInfo, + }; + + let factory = create_test_provider_factory(); + let slot = U256::from(1); + let old_value = U256::from(2); + let account = AccountInfo::default(); + let blocks = random_block_range( + &mut generators::rng(), + 0..=1, + BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() }, + ); + let mut reverts = vec![Vec::new(); 2]; + reverts[1] = vec![(ADDRESS, None, vec![(slot, old_value)])]; + let bundle = BundleState::new( + [( + ADDRESS, + Some(account.clone()), + Some(account.clone()), + HashMap::from_iter([(slot, (old_value, U256::ZERO))]), + )], + reverts, + [], + ); + + let provider_rw = factory.provider_rw().unwrap(); + provider_rw + .append_blocks_with_state( + blocks + .into_iter() + .map(|block| block.try_recover().expect("failed to seal block with senders")) + .collect(), + &ExecutionOutcome { bundle, first_block: 0, ..Default::default() }, + Default::default(), + ) + .unwrap(); + provider_rw.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(1)).unwrap(); + provider_rw.commit().unwrap(); + + let db = factory.provider().unwrap(); + let hashed_address = keccak256(ADDRESS); + assert!(db.tx_ref().get::(hashed_address).unwrap().is_none()); + + let mut destroyed_bundle = BundleState::default(); + destroyed_bundle.state.insert( + ADDRESS, + BundleAccount::new(Some(account), None, Default::default(), AccountStatus::Destroyed), + ); + let provider = HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()); + let hashed_state = provider.hashed_post_state(&destroyed_bundle).unwrap(); + + assert_eq!( + hashed_state.storages[&hashed_address].storage[&keccak256(B256::from(slot))], + U256::ZERO + ); + } + + #[test] + fn newly_created_destroyed_account_skips_historical_overlay() { + use reth_storage_api::HashedPostStateProvider; + use revm::database::{AccountStatus, BundleAccount, BundleState}; + + let factory = create_test_provider_factory(); + let db = factory.provider().unwrap(); + let mut bundle = BundleState::default(); + bundle.state.insert( + ADDRESS, + BundleAccount::new(None, None, Default::default(), AccountStatus::Destroyed), + ); + + let provider = HistoricalStateProviderRef::new(&db, 1, OverlayManager::default()); + let hashed_state = provider.hashed_post_state(&bundle).unwrap(); + + assert!(hashed_state.storages.is_empty()); + } + #[test] fn test_needs_prev_shard_check() { // Only needs check when rank == 0 and found_block != block_number diff --git a/crates/storage/provider/src/providers/state/latest.rs b/crates/storage/provider/src/providers/state/latest.rs index 1538a5f713a..9d402ef9b4c 100644 --- a/crates/storage/provider/src/providers/state/latest.rs +++ b/crates/storage/provider/src/providers/state/latest.rs @@ -9,7 +9,7 @@ use reth_storage_api::{ }; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use reth_trie::{ - hashed_cursor::HashedPostStateCursorFactory, + hashed_cursor::{zero_destroyed_account_storage, HashedPostStateCursorFactory}, proof::{Proof, StorageProof}, trie_cursor::InMemoryTrieCursorFactory, updates::TrieUpdates, @@ -252,8 +252,18 @@ impl StateProofProvider } impl HashedPostStateProvider for LatestStateProviderRef<'_, Provider> { - fn hashed_post_state(&self, bundle_state: &revm::database::BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::(bundle_state.state()) + fn hashed_post_state( + &self, + bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + let mut hashed_state = + HashedPostState::from_bundle_state::(bundle_state.state()); + zero_destroyed_account_storage( + &reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()), + bundle_state.state(), + &mut hashed_state, + )?; + Ok(hashed_state) } } diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/provider/src/providers/state/overlay.rs index 1d4aef1902f..52316b08b27 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/provider/src/providers/state/overlay.rs @@ -290,7 +290,7 @@ mod tests { .with_accounts([(hashed_address, Some(Account::default()))]) .with_storages([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot, U256::from(id))]), + HashedStorage::from_iter([(hashed_slot, U256::from(id))]), )]) .into_sorted(); let trie_updates = TrieUpdatesSorted::new( diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 68244202e1d..3fa34190c1a 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -1088,8 +1088,11 @@ where impl HashedPostStateProvider for MockEthProvider { - fn hashed_post_state(&self, _state: &revm::database::BundleState) -> HashedPostState { - HashedPostState::default() + fn hashed_post_state( + &self, + _bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + Ok(HashedPostState::default()) } } diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index d3e6f36be45..6e17e233625 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -2,9 +2,8 @@ use crate::{ AccountReader, BalProvider, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, - DatabaseProviderFactory, HashedPostStateProvider, PruneCheckpointReader, - RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, StateRangeProviderFactory, - StateReader, StaticFileProviderFactory, + DatabaseProviderFactory, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, + StateProviderFactory, StateRangeProviderFactory, StateReader, StaticFileProviderFactory, }; use reth_chain_state::{ CanonStateSubscriptions, ForkChoiceSubscriptions, PersistedBlockSubscriptions, @@ -36,7 +35,6 @@ pub trait FullProvider: + StateProviderFactory + StateRangeProviderFactory + StateReader - + HashedPostStateProvider + ChainSpecProvider + ChangeSetReader + StorageChangeSetReader @@ -74,7 +72,6 @@ impl FullProvider for T where + StateProviderFactory + StateRangeProviderFactory + StateReader - + HashedPostStateProvider + ChainSpecProvider + ChangeSetReader + StorageChangeSetReader diff --git a/crates/storage/provider/src/writer/mod.rs b/crates/storage/provider/src/writer/mod.rs index dcc1d9bb738..6f0f30db36a 100644 --- a/crates/storage/provider/src/writer/mod.rs +++ b/crates/storage/provider/src/writer/mod.rs @@ -929,13 +929,21 @@ mod tests { let overlay_root = if is_v2 { TestStateRoot::<_, PackedKeyAdapter>::overlay_root( tx, - &provider_factory.hashed_post_state(&state.bundle_state).into_sorted(), + &provider_rw + .latest() + .hashed_post_state(&state.bundle_state) + .unwrap() + .into_sorted(), ) .unwrap() } else { TestStateRoot::<_, LegacyKeyAdapter>::overlay_root( tx, - &provider_factory.hashed_post_state(&state.bundle_state).into_sorted(), + &provider_rw + .latest() + .hashed_post_state(&state.bundle_state) + .unwrap() + .into_sorted(), ) .unwrap() }; @@ -1123,7 +1131,6 @@ mod tests { // insert initial account storage let init_storage = HashedStorage::from_iter( - false, [ "50000000000000000000000000000004253371b55351a08cb3267d4d265530b6", "512428ed685fff57294d1a9cbb147b18ae5db9cf6ae4b312fa1946ba0561882e", @@ -1155,8 +1162,7 @@ mod tests { .unwrap(); // destroy the storage and re-create with new slots - let updated_storage = HashedStorage::from_iter( - true, + let mut updated_storage = HashedStorage::from_iter( [ "00deb8486ad8edccfdedfc07109b3667b38a03a8009271aac250cce062d90917", "88d233b7380bb1bcdc866f6871c94685848f54cf0ee033b1480310b4ddb75fc9", @@ -1164,6 +1170,7 @@ mod tests { .into_iter() .map(|str| (B256::from_str(str).unwrap(), U256::from(1))), ); + updated_storage.wiped = true; let mut state = HashedPostState::default(); state.storages.insert(hashed_address, updated_storage.clone()); provider_rw.write_hashed_state(&state.clone().into_sorted()).unwrap(); diff --git a/crates/storage/rpc-provider/src/lib.rs b/crates/storage/rpc-provider/src/lib.rs index 0e248a9a030..73f70dd82ae 100644 --- a/crates/storage/rpc-provider/src/lib.rs +++ b/crates/storage/rpc-provider/src/lib.rs @@ -1316,8 +1316,18 @@ where N: Network, Node: NodeTypes, { - fn hashed_post_state(&self, bundle_state: &revm::database::BundleState) -> HashedPostState { - HashedPostState::from_bundle_state::(bundle_state.state()) + fn hashed_post_state( + &self, + bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + if bundle_state + .state() + .values() + .any(|account| account.was_destroyed() && account.original_info.is_some()) + { + return Err(ProviderError::UnsupportedProvider) + } + Ok(HashedPostState::from_bundle_state::(bundle_state.state())) } } diff --git a/crates/storage/storage-api/src/macros.rs b/crates/storage/storage-api/src/macros.rs index 7844d9c3d65..7ef599a011c 100644 --- a/crates/storage/storage-api/src/macros.rs +++ b/crates/storage/storage-api/src/macros.rs @@ -62,7 +62,7 @@ macro_rules! delegate_provider_impls { fn witness(&self, input: reth_trie::TrieInput, target: reth_trie::HashedPostState, mode: reth_trie::ExecutionWitnessMode) -> reth_storage_api::errors::provider::ProviderResult>; } HashedPostStateProvider $(where [$($generics)*])? { - fn hashed_post_state(&self, bundle_state: &revm::database::BundleState) -> reth_trie::HashedPostState; + fn hashed_post_state(&self, bundle_state: &revm::database::BundleState) -> reth_storage_api::errors::provider::ProviderResult; } ); } diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 49a81d4ff26..b5f356a6f89 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -534,8 +534,11 @@ impl StateProofProvider for NoopProvider HashedPostStateProvider for NoopProvider { - fn hashed_post_state(&self, _bundle_state: &revm::database::BundleState) -> HashedPostState { - HashedPostState::default() + fn hashed_post_state( + &self, + _bundle_state: &revm::database::BundleState, + ) -> ProviderResult { + Ok(HashedPostState::default()) } } diff --git a/crates/storage/storage-api/src/state.rs b/crates/storage/storage-api/src/state.rs index 23281a38ecf..b099295e9ed 100644 --- a/crates/storage/storage-api/src/state.rs +++ b/crates/storage/storage-api/src/state.rs @@ -97,8 +97,12 @@ impl AccountInfoReader for T {} /// Trait that provides the hashed state from various sources. #[auto_impl(&, Arc, Box)] pub trait HashedPostStateProvider { - /// Returns the `HashedPostState` of the provided [`BundleState`]. - fn hashed_post_state(&self, bundle_state: &BundleState) -> HashedPostState; + /// Returns the [`HashedPostState`] of the provided [`BundleState`], materializing zero-valued + /// updates for parent storage of accounts that were destroyed but remain in the post-state. + /// + /// Providers backed by an exact parent-state view also materialize terminally destroyed + /// accounts so the result can be persisted without storage wipe markers. + fn hashed_post_state(&self, bundle_state: &BundleState) -> ProviderResult; } /// Trait for reading bytecode associated with a given code hash. diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index e894a25b0a2..b66f43d7840 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -509,7 +509,7 @@ mod tests { .with_accounts([(hashed_address, Some(Account::default()))]) .with_storages([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot, U256::from(id))]), + HashedStorage::from_iter([(hashed_slot, U256::from(id))]), )]) .into_sorted(); let trie_updates = TrieUpdatesSorted::new( diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index f824935a0b7..b53e79b748d 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -678,7 +678,7 @@ mod tests { .with_accounts([(hashed_address, Some(Account::default()))]) .with_storages([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot, U256::from(id))]), + HashedStorage::from_iter([(hashed_slot, U256::from(id))]), )]) .into_sorted(); diff --git a/crates/trie/common/src/hashed_state.rs b/crates/trie/common/src/hashed_state.rs index 89a7380504d..ecfbea13172 100644 --- a/crates/trie/common/src/hashed_state.rs +++ b/crates/trie/common/src/hashed_state.rs @@ -20,7 +20,7 @@ use reth_primitives_traits::Account; #[cfg(feature = "rayon")] use rayon::prelude::{FromParallelIterator, IntoParallelIterator, ParallelIterator}; -use revm::database::{AccountStatus, BundleAccount}; +use revm::database::BundleAccount; /// In-memory hashed state that stores account and storage changes with keccak256-hashed keys in /// hash maps. @@ -55,7 +55,6 @@ impl HashedPostState { let hashed_address = KH::hash_key(address); let hashed_account = account.info.as_ref().map(Into::into); let hashed_storage = HashedStorage::from_iter( - false, account .storage .iter() @@ -442,17 +441,14 @@ impl HashedStorage { } /// Create new hashed storage from iterator. - pub fn from_iter(wiped: bool, iter: impl IntoIterator) -> Self { - Self { wiped, storage: HashMap::from_iter(iter) } + #[expect(clippy::should_implement_trait)] + pub fn from_iter(iter: impl IntoIterator) -> Self { + Self { wiped: false, storage: HashMap::from_iter(iter) } } - /// Create new hashed storage from account status and plain storage. - pub fn from_plain_storage<'a>( - status: AccountStatus, - storage: impl IntoIterator, - ) -> Self { + /// Create new hashed storage from plain storage. + pub fn from_plain_storage<'a>(storage: impl IntoIterator) -> Self { Self::from_iter( - status.was_destroyed(), storage.into_iter().map(|(key, value)| (keccak256(B256::from(*key)), *value)), ) } @@ -999,7 +995,7 @@ mod tests { use crate::KeccakKeyHasher; use alloy_primitives::Bytes; use revm::{ - database::{states::StorageSlot, StorageWithOriginalValues}, + database::{states::StorageSlot, AccountStatus, StorageWithOriginalValues}, state::{AccountInfo, Bytecode}, }; @@ -1024,17 +1020,17 @@ mod tests { let original_slot_value = U256::from(123); let mut hashed_state = HashedPostState::default().with_storages([( hashed_address, - HashedStorage::from_iter( - false, - [(hashed_slot, original_slot_value), (hashed_slot2, original_slot_value)], - ), + HashedStorage::from_iter([ + (hashed_slot, original_slot_value), + (hashed_slot2, original_slot_value), + ]), )]); // Update single slot value let updated_slot_value = U256::from(321); let extension = HashedPostState::default().with_storages([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot, updated_slot_value)]), + HashedStorage::from_iter([(hashed_slot, updated_slot_value)]), )]); hashed_state.extend(extension); @@ -1061,7 +1057,7 @@ mod tests { // Reinitialize single slot value hashed_state.extend(HashedPostState::default().with_storages([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot, original_slot_value)]), + HashedStorage::from_iter([(hashed_slot, original_slot_value)]), )])); let account_storage = hashed_state.storages.get(&hashed_address); assert_eq!( @@ -1074,7 +1070,7 @@ mod tests { // Reinitialize single slot value hashed_state.extend(HashedPostState::default().with_storages([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot2, updated_slot_value)]), + HashedStorage::from_iter([(hashed_slot2, updated_slot_value)]), )])); let account_storage = hashed_state.storages.get(&hashed_address); assert_eq!( @@ -1923,7 +1919,7 @@ mod tests { let slot2 = B256::random(); let slot3 = B256::random(); - let mut storage = HashedStorage::from_iter(false, [(slot1, U256::from(100))]); + let mut storage = HashedStorage::from_iter([(slot1, U256::from(100))]); let sorted = HashedStorageSorted { storage_slots: vec![(slot2, U256::from(200)), (slot3, U256::ZERO)], @@ -1945,7 +1941,7 @@ mod tests { let slot1 = B256::random(); let slot2 = B256::random(); - let mut storage = HashedStorage::from_iter(false, [(slot1, U256::from(100))]); + let mut storage = HashedStorage::from_iter([(slot1, U256::from(100))]); let sorted = HashedStorageSorted { storage_slots: vec![(slot2, U256::from(200))], wiped: true }; diff --git a/crates/trie/common/src/trie_data.rs b/crates/trie/common/src/trie_data.rs index 67ae6ac5f80..c52c26e629f 100644 --- a/crates/trie/common/src/trie_data.rs +++ b/crates/trie/common/src/trie_data.rs @@ -373,7 +373,7 @@ mod tests { .with_accounts([(hashed_address, Some(Account::default()))]) .with_storages([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot, U256::from(1))]), + HashedStorage::from_iter([(hashed_slot, U256::from(1))]), )]); let (deferred, task) = diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index c97dd2ca7a9..216ffab1fb2 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -393,7 +393,7 @@ mod tests { hashed_state.accounts.insert(B256::from(U256::from(2)), None); hashed_state.storages.insert( B256::from(U256::from(1)), - HashedStorage::from_iter(false, [(B256::from(U256::from(3)), U256::from(30))]), + HashedStorage::from_iter([(B256::from(U256::from(3)), U256::from(30))]), ); let sorted = hashed_state.into_sorted(); diff --git a/crates/trie/db/tests/witness.rs b/crates/trie/db/tests/witness.rs index 6f7cd9a81a8..38d55688ff3 100644 --- a/crates/trie/db/tests/witness.rs +++ b/crates/trie/db/tests/witness.rs @@ -85,7 +85,7 @@ fn includes_empty_node_preimage() { accounts: HashMap::from_iter([(hashed_address, Some(Account::default()))]), storages: HashMap::from_iter([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot, U256::from(1))]), + HashedStorage::from_iter([(hashed_slot, U256::from(1))]), )]), }) .unwrap(); @@ -104,7 +104,7 @@ fn includes_empty_node_preimage() { accounts: HashMap::from_iter([(hashed_address, Some(Account::default()))]), storages: HashMap::from_iter([( hashed_address, - HashedStorage::from_iter(false, [(hashed_slot, U256::from(1))]), + HashedStorage::from_iter([(hashed_slot, U256::from(1))]), )]), }) .unwrap(); @@ -142,19 +142,15 @@ fn includes_nodes_for_destroyed_storage_nodes() { )])) .unwrap(); - let witness = - TrieWitness::new( - DatabaseTrieCursorFactory::<_, A>::new(provider.tx_ref()), - DatabaseHashedCursorFactory::new(provider.tx_ref()), - ) - .compute(HashedPostState { - accounts: HashMap::from_iter([(hashed_address, Some(Account::default()))]), - storages: HashMap::from_iter([( - hashed_address, - HashedStorage::from_iter(true, []), - )]), // destroyed - }) - .unwrap(); + let witness = TrieWitness::new( + DatabaseTrieCursorFactory::<_, A>::new(provider.tx_ref()), + DatabaseHashedCursorFactory::new(provider.tx_ref()), + ) + .compute(HashedPostState { + accounts: HashMap::from_iter([(hashed_address, Some(Account::default()))]), + storages: HashMap::from_iter([(hashed_address, HashedStorage::new(true))]), // destroyed + }) + .unwrap(); assert!(witness.contains_key(&state_root)); for node in multiproof.account_subtree.values() { assert_eq!(witness.get(&keccak256(node)), Some(node)); @@ -205,7 +201,6 @@ fn correctly_decodes_branch_node_values() { storages: HashMap::from_iter([( hashed_address, HashedStorage::from_iter( - false, [hashed_slot1, hashed_slot2].map(|hashed_slot| (hashed_slot, U256::from(2))), ), )]), @@ -347,10 +342,10 @@ fn canonical_mode_handles_mixed_storage_inserts_and_removals() { accounts: HashMap::from_iter([(hashed_address, Some(Account::default()))]), storages: HashMap::from_iter([( hashed_address, - HashedStorage::from_iter( - false, - [(removed_slot, U256::ZERO), (inserted_slot, U256::from(3))], - ), + HashedStorage::from_iter([ + (removed_slot, U256::ZERO), + (inserted_slot, U256::from(3)), + ]), )]), }; diff --git a/crates/trie/parallel/src/state_root_task.rs b/crates/trie/parallel/src/state_root_task.rs index d3a305a6cd3..eabc3770649 100644 --- a/crates/trie/parallel/src/state_root_task.rs +++ b/crates/trie/parallel/src/state_root_task.rs @@ -516,7 +516,7 @@ pub fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostState { if !destroyed && changed_storage_iter.peek().is_some() { hashed_state .storages - .insert(hashed_address, HashedStorage::from_iter(false, changed_storage_iter)); + .insert(hashed_address, HashedStorage::from_iter(changed_storage_iter)); } } } diff --git a/crates/trie/trie/src/hashed_cursor/mod.rs b/crates/trie/trie/src/hashed_cursor/mod.rs index 3e5a0452975..ea526cc769c 100644 --- a/crates/trie/trie/src/hashed_cursor/mod.rs +++ b/crates/trie/trie/src/hashed_cursor/mod.rs @@ -80,6 +80,7 @@ pub trait HashedStorageCursor: HashedCursor { /// Materializes storage deletions for destroyed accounts as explicit zero-valued slot updates. /// +/// Accounts absent from the bundle pre-state are skipped because they cannot have parent storage. /// Final bundle values take precedence so that destroy-then-recreate transitions retain storage /// written by the recreated account. pub fn zero_destroyed_account_storage<'a>( @@ -89,7 +90,7 @@ pub fn zero_destroyed_account_storage<'a>( ) -> Result<(), DatabaseError> { let mut destroyed_accounts = accounts .into_iter() - .filter(|(_, account)| account.was_destroyed()) + .filter(|(_, account)| account.was_destroyed() && account.original_info.is_some()) .map(|(address, _)| keccak256(address)); let Some(mut hashed_address) = destroyed_accounts.next() else { return Ok(()) }; let mut cursor = cursor_factory.hashed_storage_cursor(hashed_address)?; @@ -110,3 +111,25 @@ pub fn zero_destroyed_account_storage<'a>( Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + use revm::database::AccountStatus; + + #[test] + fn zero_destroyed_storage_skips_new_accounts() { + let address = Address::with_last_byte(1); + let account = BundleAccount::new(None, None, Default::default(), AccountStatus::Destroyed); + let mut hashed_state = HashedPostState::default(); + + zero_destroyed_account_storage( + &mock::MockHashedCursorFactory::default(), + [(&address, &account)], + &mut hashed_state, + ) + .unwrap(); + + assert!(hashed_state.storages.is_empty()); + } +} diff --git a/crates/trie/trie/src/test_utils.rs b/crates/trie/trie/src/test_utils.rs index c1871a89970..3244d1d3051 100644 --- a/crates/trie/trie/src/test_utils.rs +++ b/crates/trie/trie/src/test_utils.rs @@ -119,8 +119,7 @@ impl TrieTestHarness { prefix_set.insert(Nibbles::unpack(hashed_slot)); } - let hashed_storage = - HashedStorage::from_iter(false, changeset.iter().map(|(&k, &v)| (k, v))); + let hashed_storage = HashedStorage::from_iter(changeset.iter().map(|(&k, &v)| (k, v))); let overlay = HashedPostStateSorted::new( Vec::new(), once((self.hashed_address(), hashed_storage.into_sorted())).collect(), diff --git a/testing/ef-tests/src/cases/blockchain_test.rs b/testing/ef-tests/src/cases/blockchain_test.rs index a497fa408de..2c54e3870e3 100644 --- a/testing/ef-tests/src/cases/blockchain_test.rs +++ b/testing/ef-tests/src/cases/blockchain_test.rs @@ -16,14 +16,13 @@ use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::{ParallelBridgeBuffered, RecoveredBlock, SealedBlock}; use reth_provider::{ test_utils::create_test_provider_factory_with_chain_spec, BlockWriter, DatabaseProviderFactory, - ExecutionOutcome, HistoryWriter, OriginalValuesKnown, StateWriteConfig, StateWriter, - StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, StorageSettingsCache, + ExecutionOutcome, HashedPostStateProvider, HistoryWriter, OriginalValuesKnown, + StateWriteConfig, StateWriter, StaticFileProviderFactory, StaticFileSegment, StaticFileWriter, + StorageSettingsCache, }; use reth_revm::database::StateProviderDatabase; -use reth_trie::{ - hashed_cursor::zero_destroyed_account_storage, HashedPostState, KeccakKeyHasher, StateRoot, -}; -use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; +use reth_trie::StateRoot; +use reth_trie_db::DatabaseStateRoot; use std::{ collections::BTreeMap, fs, @@ -258,14 +257,9 @@ fn run_case(case: &BlockchainTest) -> Result<(), Error> { .map_err(|err| Error::block_failed(block_number, err))?; // Compute and check the post state root - let mut hashed_state = - HashedPostState::from_bundle_state::(output.state.state()); - zero_destroyed_account_storage( - &DatabaseHashedCursorFactory::new(provider.tx_ref()), - output.state.state(), - &mut hashed_state, - ) - .map_err(|err| Error::block_failed(block_number, err))?; + let hashed_state = state_provider + .hashed_post_state(&output.state) + .map_err(|err| Error::block_failed(block_number, err))?; let sorted = hashed_state.clone_into_sorted(); let (computed_state_root, _) = reth_trie_db::with_adapter!(provider, |A| { StateRoot::, _>::overlay_root_with_updates( From 8104ed1dfd4e446a86da5e0e82bfc24195f6a56e Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 5 Aug 2026 12:34:55 +0200 Subject: [PATCH 094/148] refactor(storage): remove ConsistentDbView (#26581) Co-authored-by: Brian Picciano --- .../provider/src/providers/consistent_view.rs | 222 ------------------ crates/storage/provider/src/providers/mod.rs | 3 - 2 files changed, 225 deletions(-) delete mode 100644 crates/storage/provider/src/providers/consistent_view.rs diff --git a/crates/storage/provider/src/providers/consistent_view.rs b/crates/storage/provider/src/providers/consistent_view.rs deleted file mode 100644 index 653b3659cb5..00000000000 --- a/crates/storage/provider/src/providers/consistent_view.rs +++ /dev/null @@ -1,222 +0,0 @@ -use crate::{BlockNumReader, DatabaseProviderFactory, HeaderProvider}; -use alloy_primitives::B256; -pub use reth_storage_errors::provider::ConsistentViewError; -use reth_storage_errors::provider::ProviderResult; - -/// A consistent view over state in the database. -/// -/// View gets initialized with the latest or provided tip. -/// Upon every attempt to create a database provider, the view will -/// perform a consistency check of current tip against the initial one. -/// -/// ## Usage -/// -/// The view should only be used outside of staged-sync. -/// Otherwise, any attempt to create a provider will result in [`ConsistentViewError::Syncing`]. -/// -/// When using the view, the consumer should either -/// 1) have a failover for when the state changes and handle [`ConsistentViewError::Inconsistent`] -/// appropriately. -/// 2) be sure that the state does not change. -#[derive(Clone, Debug)] -pub struct ConsistentDbView { - factory: Factory, - tip: Option<(B256, u64)>, -} - -impl ConsistentDbView -where - Factory: DatabaseProviderFactory, -{ - /// Creates new consistent database view. - pub const fn new(factory: Factory, tip: Option<(B256, u64)>) -> Self { - Self { factory, tip } - } - - /// Creates new consistent database view with latest tip. - pub fn new_with_latest_tip(provider: Factory) -> ProviderResult { - let provider_ro = provider.database_provider_ro()?; - let last_num = provider_ro.last_block_number()?; - let tip = provider_ro.sealed_header(last_num)?.map(|h| (h.hash(), last_num)); - Ok(Self::new(provider, tip)) - } - - /// Creates new read-only provider and performs consistency checks on the current tip. - pub fn provider_ro(&self) -> ProviderResult { - // Create a new provider. - let provider_ro = self.factory.database_provider_ro()?; - - // Check that the currently stored tip is included on-disk. - // This means that the database may have moved, but the view was not reorged. - // - // NOTE: We must use `sealed_header` with the block number here, because if we are using - // the consistent view provider while we're persisting blocks, we may enter a race - // condition. Recall that we always commit to static files first, then the database, and - // that block hash to block number indexes are contained in the database. If we were to - // fetch the block by hash while we're persisting, the following situation may occur: - // - // 1. Persistence appends the latest block to static files. - // 2. We initialize the consistent view provider, which fetches based on `last_block_number` - // and `sealed_header`, which both check static files, setting the tip to the newly - // committed block. - // 3. We attempt to fetch a header by hash, using for example the `header` method. This - // checks the database first, to fetch the number corresponding to the hash. Because the - // database has not been committed yet, this fails, and we return - // `ConsistentViewError::Reorged`. - // 4. Some time later, the database commits. - // - // To ensure this doesn't happen, we just have to make sure that we fetch from the same - // data source that we used during initialization. In this case, that is static files - if let Some((hash, number)) = self.tip && - provider_ro.sealed_header(number)?.is_none_or(|header| header.hash() != hash) - { - return Err(ConsistentViewError::Reorged { block: hash }.into()) - } - - Ok(provider_ro) - } -} - -#[cfg(test)] -mod tests { - use reth_errors::ProviderError; - use std::str::FromStr; - - use super::*; - use crate::{test_utils::create_test_provider_factory, BlockWriter}; - use alloy_primitives::Bytes; - use assert_matches::assert_matches; - use reth_chainspec::{ChainSpecProvider, EthChainSpec}; - use reth_ethereum_primitives::{Block, BlockBody}; - use reth_primitives_traits::{block::TestBlock, RecoveredBlock, SealedBlock}; - - #[test] - fn test_consistent_view_extend() { - let provider_factory = create_test_provider_factory(); - - let genesis_block = SealedBlock::::seal_parts( - provider_factory.chain_spec().genesis_header().clone(), - BlockBody::default(), - ); - let genesis_hash: B256 = genesis_block.hash(); - let genesis_block = RecoveredBlock::new_sealed(genesis_block, vec![]); - - // insert the block - let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(&genesis_block).unwrap(); - provider_rw.commit().unwrap(); - - // create a consistent view provider and check that a ro provider can be made - let view = ConsistentDbView::new_with_latest_tip(provider_factory.clone()).unwrap(); - - // ensure successful creation of a read-only provider. - assert_matches!(view.provider_ro(), Ok(_)); - - // generate a block that extends the genesis - let mut block = Block::default(); - block.header_mut().parent_hash = genesis_hash; - block.header_mut().number = 1; - let sealed_block = SealedBlock::seal_slow(block); - let recovered_block = RecoveredBlock::new_sealed(sealed_block, vec![]); - - // insert the block - let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(&recovered_block).unwrap(); - provider_rw.commit().unwrap(); - - // ensure successful creation of a read-only provider, based on this new db state. - assert_matches!(view.provider_ro(), Ok(_)); - - // generate a block that extends that block - let mut block = Block::default(); - block.header_mut().parent_hash = genesis_hash; - block.header_mut().number = 2; - let sealed_block = SealedBlock::seal_slow(block); - let recovered_block = RecoveredBlock::new_sealed(sealed_block, vec![]); - - // insert the block - let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(&recovered_block).unwrap(); - provider_rw.commit().unwrap(); - - // check that creation of a read-only provider still works - assert_matches!(view.provider_ro(), Ok(_)); - } - - #[test] - fn test_consistent_view_remove() { - let provider_factory = create_test_provider_factory(); - - let genesis_block = SealedBlock::::seal_parts( - provider_factory.chain_spec().genesis_header().clone(), - BlockBody::default(), - ); - let genesis_hash: B256 = genesis_block.hash(); - let genesis_block = RecoveredBlock::new_sealed(genesis_block, vec![]); - - // insert the block - let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(&genesis_block).unwrap(); - provider_rw.commit().unwrap(); - - // create a consistent view provider and check that a ro provider can be made - let view = ConsistentDbView::new_with_latest_tip(provider_factory.clone()).unwrap(); - - // ensure successful creation of a read-only provider. - assert_matches!(view.provider_ro(), Ok(_)); - - // generate a block that extends the genesis - let mut block = Block::default(); - block.header_mut().parent_hash = genesis_hash; - block.header_mut().number = 1; - let sealed_block = SealedBlock::seal_slow(block); - let recovered_block = RecoveredBlock::new_sealed(sealed_block.clone(), vec![]); - - // insert the block - let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(&recovered_block).unwrap(); - provider_rw.commit().unwrap(); - - // create a second consistent view provider and check that a ro provider can be made - let view = ConsistentDbView::new_with_latest_tip(provider_factory.clone()).unwrap(); - let initial_tip_hash = sealed_block.hash(); - - // ensure successful creation of a read-only provider, based on this new db state. - assert_matches!(view.provider_ro(), Ok(_)); - - // remove the block above the genesis block - let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.remove_blocks_above(0).unwrap(); - provider_rw.commit().unwrap(); - - // ensure unsuccessful creation of a read-only provider, based on this new db state. - let Err(ProviderError::ConsistentView(boxed_consistent_view_err)) = view.provider_ro() - else { - panic!("expected reorged consistent view error, got success"); - }; - let unboxed = *boxed_consistent_view_err; - assert_eq!(unboxed, ConsistentViewError::Reorged { block: initial_tip_hash }); - - // generate a block that extends the genesis with a different hash - let mut block = Block::default(); - block.header_mut().parent_hash = genesis_hash; - block.header_mut().number = 1; - block.header_mut().extra_data = - Bytes::from_str("6a6f75726e657920746f20697468616361").unwrap(); - let sealed_block = SealedBlock::seal_slow(block); - let recovered_block = RecoveredBlock::new_sealed(sealed_block, vec![]); - - // reinsert the block at the same height, but with a different hash - let provider_rw = provider_factory.provider_rw().unwrap(); - provider_rw.insert_block(&recovered_block).unwrap(); - provider_rw.commit().unwrap(); - - // ensure unsuccessful creation of a read-only provider, based on this new db state. - let Err(ProviderError::ConsistentView(boxed_consistent_view_err)) = view.provider_ro() - else { - panic!("expected reorged consistent view error, got success"); - }; - let unboxed = *boxed_consistent_view_err; - assert_eq!(unboxed, ConsistentViewError::Reorged { block: initial_tip_hash }); - } -} diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index ac6a12fef5c..d41d335e170 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -23,9 +23,6 @@ pub use state::{ overlay::{OverlayStateProvider, OverlayStateProviderFactory}, }; -mod consistent_view; -pub use consistent_view::{ConsistentDbView, ConsistentViewError}; - mod blockchain_provider; pub use blockchain_provider::{BlockchainProvider, SNAPSHOT_STATE_RETENTION}; From 859e31a30d0ad407273b74230a7990a2b7cca5ab Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 5 Aug 2026 15:52:51 +0200 Subject: [PATCH 095/148] refactor(provider): remove unused AccountReader bounds (#26591) --- crates/rpc/rpc-builder/src/lib.rs | 11 +-- .../src/providers/blockchain_provider.rs | 11 +-- .../provider/src/providers/consistent.rs | 68 +++---------------- crates/storage/provider/src/traits/full.rs | 8 +-- 4 files changed, 16 insertions(+), 82 deletions(-) diff --git a/crates/rpc/rpc-builder/src/lib.rs b/crates/rpc/rpc-builder/src/lib.rs index c3dd3392168..3e818dabc56 100644 --- a/crates/rpc/rpc-builder/src/lib.rs +++ b/crates/rpc/rpc-builder/src/lib.rs @@ -56,8 +56,7 @@ use reth_rpc_eth_types::{receipt::EthReceiptConverter, EthConfig, EthSubscriptio use reth_rpc_layer::{AuthLayer, Claims, CompressionLayer, JwtAuthValidator, JwtSecret}; pub use reth_rpc_server_types::RethRpcModule; use reth_storage_api::{ - AccountReader, BlockReader, ChangeSetReader, FullRpcProvider, NodePrimitivesProvider, - StateProviderFactory, + BlockReader, ChangeSetReader, FullRpcProvider, NodePrimitivesProvider, StateProviderFactory, }; use reth_tasks::{pool::BlockingTaskGuard, Runtime}; use reth_tokio_util::EventSender; @@ -316,7 +315,6 @@ where + CanonStateSubscriptions + ForkChoiceSubscriptions

+ PersistedBlockSubscriptions - + AccountReader + ChangeSetReader, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, @@ -663,8 +661,7 @@ where Block = N::Block, Receipt = N::Receipt, Transaction = N::SignedTx, - > + AccountReader - + ChangeSetReader + > + ChangeSetReader + CanonStateSubscriptions + ForkChoiceSubscriptions
+ PersistedBlockSubscriptions, @@ -781,8 +778,7 @@ where Header = N::BlockHeader, Transaction = N::SignedTx, Receipt = N::Receipt, - > + AccountReader - + ChangeSetReader, + > + ChangeSetReader, Network: NetworkInfo + Peers + Clone + 'static, EthApi: EthApiTypes, EvmConfig: ConfigureEvm, @@ -862,7 +858,6 @@ where + CanonStateSubscriptions + ForkChoiceSubscriptions
+ PersistedBlockSubscriptions - + AccountReader + ChangeSetReader, Pool: TransactionPool + Clone + 'static, Network: NetworkInfo + Peers + Clone + 'static, diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index c31b47cbb38..6085fbc1dda 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -3,8 +3,8 @@ use crate::{ ConsistentProvider, OverlayStateProvider, OverlayStateProviderFactory, ProviderNodeTypes, RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut, }, - AccountReader, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, - BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, + BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, + BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, DatabaseProviderFactory, HeaderProvider, ProviderError, ProviderFactory, ProviderHeader, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RocksDBProviderFactory, @@ -980,13 +980,6 @@ impl ChangeSetReader for BlockchainProvider { } } -impl AccountReader for BlockchainProvider { - /// Get basic account information. - fn basic_account(&self, address: &Address) -> ProviderResult> { - self.consistent_provider()?.basic_account(address) - } -} - impl StateReader for BlockchainProvider { type Receipt = ReceiptTy; diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index 2fbd3920105..863b57ae534 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -1,11 +1,10 @@ use super::{DatabaseProviderRO, ProviderFactory, ProviderNodeTypes}; use crate::{ providers::{StaticFileProvider, StaticFileProviderRWRefMut}, - to_range, AccountReader, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, - BlockReaderIdExt, BlockSource, ChainSpecProvider, ChangeSetReader, HeaderProvider, - ProviderError, ProviderHeader, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, - StageCheckpointReader, StateReader, StaticFileProviderFactory, TransactionVariant, - TransactionsProvider, + to_range, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, + BlockSource, ChainSpecProvider, ChangeSetReader, HeaderProvider, ProviderError, ProviderHeader, + PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, + StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; use alloy_consensus::{ transaction::{TransactionMeta, TxHashRef}, @@ -13,20 +12,20 @@ use alloy_consensus::{ }; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag, HashOrNumber}; use alloy_primitives::{Address, BlockHash, BlockNumber, TxHash, TxNumber, B256}; -use reth_chain_state::{BlockState, CanonicalInMemoryState, MemoryOverlayStateProviderRef}; +use reth_chain_state::{BlockState, CanonicalInMemoryState}; use reth_chainspec::ChainInfo; use reth_db_api::models::{AccountBeforeTx, BlockNumberAddress, StoredBlockBodyIndices}; use reth_execution_types::ExecutionOutcome; use reth_node_types::{BlockTy, HeaderTy, ReceiptTy, TxTy}; use reth_primitives_traits::{ - Account, BlockBody, RecoveredBlock, SealedHeader, SealedOrRecoveredBlock, StorageEntry, + BlockBody, RecoveredBlock, SealedHeader, SealedOrRecoveredBlock, StorageEntry, }; use reth_prune_types::{PruneCheckpoint, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ - BlockBodyIndicesProvider, DatabaseProviderFactory, NodePrimitivesProvider, StateProvider, - StateProviderBox, StorageChangeSetReader, TryIntoHistoricalStateProvider, + BlockBodyIndicesProvider, DatabaseProviderFactory, NodePrimitivesProvider, StateProviderBox, + StorageChangeSetReader, TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::ProviderResult; use revm::database::states::PlainStorageRevert; @@ -34,7 +33,6 @@ use std::{ ops::{Add, Bound, RangeBounds, RangeInclusive, Sub}, sync::Arc, }; -use tracing::trace; /// Type that interacts with a snapshot view of the blockchain (storage and in-memory) at time of /// instantiation, EXCEPT for pending, safe and finalized block which might change while holding @@ -99,36 +97,6 @@ impl ConsistentProvider { (start, end) } - /// Storage provider for latest block - fn latest_ref<'a>(&'a self) -> ProviderResult> { - trace!(target: "providers::blockchain", "Getting latest block state provider"); - - // use latest state provider if the head state exists - if let Some(state) = &self.head_block { - trace!(target: "providers::blockchain", "Using head state for latest state provider"); - Ok(self.block_state_provider_ref(state)?.boxed()) - } else { - trace!(target: "providers::blockchain", "Using database state for latest state provider"); - Ok(self.storage_provider.latest()) - } - } - - fn history_by_block_hash_ref<'a>( - &'a self, - block_hash: BlockHash, - ) -> ProviderResult> { - trace!(target: "providers::blockchain", ?block_hash, "Getting history by block hash"); - - self.get_in_memory_or_storage_by_block( - block_hash.into(), - |_| self.storage_provider.history_by_block_hash(block_hash), - |block_state| { - let state_provider = self.block_state_provider_ref(block_state)?; - Ok(Box::new(state_provider)) - }, - ) - } - /// Fetches a range of data from both in-memory state and persistent storage while a predicate /// is met. /// @@ -242,17 +210,6 @@ impl ConsistentProvider { Ok(items) } - /// This uses a given [`BlockState`] to initialize a state provider for that block. - fn block_state_provider_ref( - &self, - state: &BlockState, - ) -> ProviderResult> { - let anchor_hash = state.anchor().hash; - let latest_historical = self.history_by_block_hash_ref(anchor_hash)?; - let in_memory = state.chain().map(|block_state| block_state.block()).collect(); - Ok(MemoryOverlayStateProviderRef::new(latest_historical, in_memory)) - } - /// Fetches data from either in-memory state or persistent storage for a range of transactions. /// /// * `fetch_from_db`: has a `DatabaseProviderRO` and the storage specific range. @@ -1514,15 +1471,6 @@ impl ChangeSetReader for ConsistentProvider { } } -impl AccountReader for ConsistentProvider { - /// Get basic account information. - fn basic_account(&self, address: &Address) -> ProviderResult> { - // use latest state provider - let state_provider = self.latest_ref()?; - state_provider.basic_account(address) - } -} - impl StateReader for ConsistentProvider { type Receipt = ReceiptTy; diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index 6e17e233625..fb0a18565a2 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -1,7 +1,7 @@ //! Helper provider traits to encapsulate all provider traits for simplicity. use crate::{ - AccountReader, BalProvider, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, + BalProvider, BlockReader, BlockReaderIdExt, ChainSpecProvider, ChangeSetReader, DatabaseProviderFactory, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, StateProviderFactory, StateRangeProviderFactory, StateReader, StaticFileProviderFactory, }; @@ -30,8 +30,7 @@ pub trait FullProvider: Block = BlockTy, Receipt = ReceiptTy, Header = HeaderTy, - > + AccountReader - + BalProvider + > + BalProvider + StateProviderFactory + StateRangeProviderFactory + StateReader @@ -67,8 +66,7 @@ impl FullProvider for T where Block = BlockTy, Receipt = ReceiptTy, Header = HeaderTy, - > + AccountReader - + BalProvider + > + BalProvider + StateProviderFactory + StateRangeProviderFactory + StateReader From ecfe7d3d68bd1c3cde7a9e38b0e482fd0f66a9b4 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:11:47 +0200 Subject: [PATCH 096/148] fix(bench): show metrics in run config (#26461) Co-authored-by: Sergei Shulepov Co-authored-by: Matthias Seitz --- .github/workflows/bench.yml | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index 956652f17e9..58f4c400bbb 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -665,6 +665,8 @@ jobs: const runPairsNote = `, run-pairs: \`${runPairs}\`${runOrder ? ` (${runOrder})` : ''}`; const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false'; const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : ''; + const metricsEnabled = '${{ steps.args.outputs.metrics }}' === 'true'; + const metricsNote = metricsEnabled ? ', metrics: `enabled`' : ''; const waitTimeVal = '${{ steps.args.outputs.wait-time }}'; const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : ''; const blockTimeVal = '${{ steps.args.outputs.block-time }}'; @@ -674,7 +676,7 @@ jobs: const featureArgsVal = '${{ steps.args.outputs.feature-args }}'; const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : ''; const blocksDesc = bigBlocks ? `${blocks} big blocks, ${warmup} warmup blocks` : `${blocks} blocks, ${warmup} warmup blocks`; - const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${modeNote}${samplyNote}${tracingChromeNote}${slackNote}${reorgNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${waitTimeNote}${blockTimeNote}${baselineArgsNote}${featureArgsNote}`; + const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${modeNote}${samplyNote}${tracingChromeNote}${slackNote}${reorgNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${metricsNote}${waitTimeNote}${blockTimeNote}${baselineArgsNote}${featureArgsNote}`; const { data: comment } = await github.rest.issues.createComment({ owner: context.repo.owner, @@ -718,6 +720,8 @@ jobs: const runPairsNote = `, run-pairs: \`${runPairs}\`${runOrder ? ` (${runOrder})` : ''}`; const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false'; const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : ''; + const metricsEnabled = '${{ steps.args.outputs.metrics }}' === 'true'; + const metricsNote = metricsEnabled ? ', metrics: `enabled`' : ''; const waitTimeVal = '${{ steps.args.outputs.wait-time }}'; const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : ''; const blockTimeVal = '${{ steps.args.outputs.block-time }}'; @@ -727,7 +731,7 @@ jobs: const featureArgsVal = '${{ steps.args.outputs.feature-args }}'; const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : ''; const blocksDesc = bigBlocks ? `${blocks} big blocks, ${warmup} warmup blocks` : `${blocks} blocks, ${warmup} warmup blocks`; - const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${modeNote}${samplyNote}${tracingChromeNote}${slackNote}${reorgNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${waitTimeNote}${blockTimeNote}${baselineArgsNote}${featureArgsNote}`; + const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${modeNote}${samplyNote}${tracingChromeNote}${slackNote}${reorgNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${metricsNote}${waitTimeNote}${blockTimeNote}${baselineArgsNote}${featureArgsNote}`; const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`; const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1; @@ -896,6 +900,8 @@ jobs: const runPairsNote = `, run-pairs: \`${runPairs}\`${runOrder ? ` (${runOrder})` : ''}`; const otlpEnabled = (process.env.BENCH_OTLP || 'false') !== 'false'; const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : ''; + const metricsEnabled = process.env.BENCH_METRICS === 'true'; + const metricsNote = metricsEnabled ? ', metrics: `enabled`' : ''; const waitTimeVal = process.env.BENCH_WAIT_TIME || ''; const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : ''; const blockTimeVal = process.env.BENCH_BLOCK_TIME || ''; @@ -905,7 +911,7 @@ jobs: const featureArgsVal = process.env.BENCH_FEATURE_ARGS || ''; const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : ''; const blocksDesc = bigBlocks ? `${blocks} big blocks, ${warmup} warmup blocks` : `${blocks} blocks, ${warmup} warmup blocks`; - const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${modeNote}${samplyNote}${tracingChromeNote}${slackNote}${reorgNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${waitTimeNote}${blockTimeNote}${baselineArgsNote}${featureArgsNote}`; + const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${modeNote}${samplyNote}${tracingChromeNote}${slackNote}${reorgNote}${balNote}${coresNote}${runPairsNote}${otlpNote}${metricsNote}${waitTimeNote}${blockTimeNote}${baselineArgsNote}${featureArgsNote}`; core.exportVariable('BENCH_CONFIG', config); const pr = process.env.BENCH_PR; From eb366066fe1add425ae62048e7f7bf58f9c0b18a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?L=C3=A9a=20Narzis?= <78718413+lean-apple@users.noreply.github.com> Date: Wed, 5 Aug 2026 16:16:09 +0200 Subject: [PATCH 097/148] chore(net): share the snap/2 slim account codec between server and client (#26587) --- Cargo.lock | 1 + crates/net/eth-wire-types/Cargo.toml | 4 + crates/net/eth-wire-types/src/snap.rs | 153 +++++++++++++++++++++++- crates/net/network/src/eth_requests.rs | 61 ++-------- crates/net/network/tests/it/snap/mod.rs | 60 +++------- crates/net/p2p/src/snap/client.rs | 78 +++++++++++- 6 files changed, 261 insertions(+), 96 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d1b7815be38..db4f92abaa0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8658,6 +8658,7 @@ dependencies = [ "alloy-hardforks 0.4.7", "alloy-primitives", "alloy-rlp", + "alloy-trie", "arbitrary", "bytes", "derive_more", diff --git a/crates/net/eth-wire-types/Cargo.toml b/crates/net/eth-wire-types/Cargo.toml index dc0058ebe9a..7d0842407f9 100644 --- a/crates/net/eth-wire-types/Cargo.toml +++ b/crates/net/eth-wire-types/Cargo.toml @@ -24,6 +24,7 @@ alloy-eip7928 = { workspace = true, features = ["rlp"], optional = true } alloy-eips.workspace = true alloy-primitives = { workspace = true, features = ["map"] } alloy-rlp = { workspace = true, features = ["derive"] } +alloy-trie.workspace = true alloy-consensus.workspace = true bytes.workspace = true @@ -68,6 +69,7 @@ std = [ "serde?/std", "thiserror/std", "reth-chainspec/std", + "alloy-trie/std", ] arbitrary = [ "reth-ethereum-primitives/arbitrary", @@ -82,6 +84,7 @@ arbitrary = [ "alloy-eips/arbitrary", "alloy-primitives/arbitrary", "reth-primitives-traits/arbitrary", + "alloy-trie/arbitrary", ] serde = [ "dep:serde", @@ -95,4 +98,5 @@ serde = [ "reth-ethereum-primitives/serde", "alloy-hardforks/serde", "alloy-eip7928?/serde", + "alloy-trie/serde", ] diff --git a/crates/net/eth-wire-types/src/snap.rs b/crates/net/eth-wire-types/src/snap.rs index 903175e40e6..1711cc77645 100644 --- a/crates/net/eth-wire-types/src/snap.rs +++ b/crates/net/eth-wire-types/src/snap.rs @@ -7,8 +7,9 @@ use crate::BlockAccessLists; use alloc::vec::Vec; -use alloy_primitives::{Bytes, B256}; +use alloy_primitives::{Bytes, B256, KECCAK256_EMPTY, U256}; use alloy_rlp::{BufMut, Decodable, Encodable, RlpDecodable, RlpEncodable}; +use alloy_trie::{TrieAccount, EMPTY_ROOT_HASH}; use reth_codecs_derive::add_arbitrary_tests; /// Supported SNAP protocol versions. @@ -113,6 +114,34 @@ pub struct AccountData { pub body: Bytes, } +impl AccountData { + /// Encodes `account` in snap/2's slim format. + pub fn from_trie_account(hash: B256, account: &TrieAccount) -> Self { + let body = alloy_rlp::encode(SlimAccountBodyRef { + nonce: account.nonce, + balance: account.balance, + storage_root: SlimAccountBodyRef::shorten(&account.storage_root, EMPTY_ROOT_HASH), + code_hash: SlimAccountBodyRef::shorten(&account.code_hash, KECCAK256_EMPTY), + }); + Self { hash, body: body.into() } + } + + /// Decodes the slim body into the account the trie leaf commits to. + /// + /// Range proofs are verified against the full encoding, so the omitted storage root and code + /// hash are restored to their defaults here. + pub fn trie_account(&self) -> alloy_rlp::Result { + let slim = alloy_rlp::decode_exact::(&self.body)?; + + Ok(TrieAccount { + nonce: slim.nonce, + balance: slim.balance, + storage_root: SlimAccountBody::restore(&slim.storage_root, EMPTY_ROOT_HASH)?, + code_hash: SlimAccountBody::restore(&slim.code_hash, KECCAK256_EMPTY)?, + }) + } +} + /// Response containing a number of consecutive accounts and the Merkle proofs for the entire range. // http://github.com/ethereum/devp2p/blob/master/caps/snap.md#accountrange-0x01 #[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)] @@ -210,6 +239,18 @@ pub struct StorageData { pub data: Bytes, } +impl StorageData { + /// Encodes a slot value as the storage trie leaf commits to it. + pub fn from_value(hash: B256, value: U256) -> Self { + Self { hash, data: alloy_rlp::encode(value).into() } + } + + /// Decodes the slot value. + pub fn value(&self) -> alloy_rlp::Result { + alloy_rlp::decode_exact(&self.data) + } +} + /// Response containing a number of consecutive storage slots for the requested account /// and optionally the merkle proofs for the last range (boundary proofs) if it only partially /// covers the storage trie. @@ -479,6 +520,53 @@ impl SnapProtocolMessage { } } +/// Like a trie account, with empty code and storage hashes omitted to reduce transfer size. +#[derive(RlpDecodable)] +struct SlimAccountBody { + /// The account's nonce. + nonce: u64, + /// The account's balance. + balance: U256, + /// Empty when the account has no storage. + storage_root: Bytes, + /// Empty when the account has no code. + code_hash: Bytes, +} + +impl SlimAccountBody { + /// Restores a dropped field to `empty`, rejecting any length the encoding never produces. + fn restore(value: &[u8], empty: B256) -> alloy_rlp::Result { + match value { + [] => Ok(empty), + _ => B256::try_from(value).map_err(|_| alloy_rlp::Error::UnexpectedLength), + } + } +} + +/// Borrowed encode twin of [`SlimAccountBody`]. +#[derive(RlpEncodable)] +struct SlimAccountBodyRef<'a> { + /// The account's nonce. + nonce: u64, + /// The account's balance. + balance: U256, + /// Empty when the account has no storage. + storage_root: &'a [u8], + /// Empty when the account has no code. + code_hash: &'a [u8], +} + +impl<'a> SlimAccountBodyRef<'a> { + /// Drops a field that holds its empty default, which is what makes the encoding slim. + fn shorten(value: &'a B256, empty: B256) -> &'a [u8] { + if *value == empty { + &[] + } else { + value.as_slice() + } + } +} + #[cfg(test)] mod tests { use super::*; @@ -789,4 +877,67 @@ mod tests { assert_eq!(msg.starting_hash.unwrap_or(B256::ZERO), B256::ZERO); assert_eq!(msg.limit_hash.unwrap_or(B256::repeat_byte(0xff)), B256::repeat_byte(0xff)); } + + fn trie_account(storage_root: B256, code_hash: B256) -> TrieAccount { + TrieAccount { nonce: 7, balance: U256::from(42), storage_root, code_hash } + } + + #[test] + fn slim_body_elides_empty_storage_and_code() { + let account = trie_account(EMPTY_ROOT_HASH, KECCAK256_EMPTY); + let encoded = AccountData::from_trie_account(B256::repeat_byte(1), &account); + + let body = SlimAccountBody::decode(&mut encoded.body.as_ref()).unwrap(); + assert!(body.storage_root.is_empty()); + assert!(body.code_hash.is_empty()); + assert_eq!(encoded.trie_account().unwrap(), account); + } + + #[test] + fn slim_body_keeps_non_default_storage_and_code() { + let account = trie_account(B256::repeat_byte(2), B256::repeat_byte(3)); + let encoded = AccountData::from_trie_account(B256::repeat_byte(1), &account); + + let body = SlimAccountBody::decode(&mut encoded.body.as_ref()).unwrap(); + assert_eq!(body.storage_root.len(), 32); + assert_eq!(body.code_hash.len(), 32); + assert_eq!(encoded.trie_account().unwrap(), account); + } + + #[test] + fn slim_body_rejects_field_lengths_the_encoding_never_produces() { + // A 16-byte field is neither an elided default nor a hash, so accepting it would let a + // peer smuggle a value that hashes differently than the one it claims to serve. + let truncated = Bytes::from_static(&[0xaa; 16]); + + assert!(SlimAccountBody::restore(&truncated, EMPTY_ROOT_HASH).is_err()); + } + + #[test] + fn slim_body_rejects_trailing_bytes() { + let account = trie_account(EMPTY_ROOT_HASH, KECCAK256_EMPTY); + let mut encoded = AccountData::from_trie_account(B256::repeat_byte(1), &account); + encoded.body = [encoded.body.as_ref(), &[0x00]].concat().into(); + + assert!(encoded.trie_account().is_err()); + } + + #[test] + fn storage_data_carries_the_trie_leaf_encoding() { + let value = U256::from(1234); + let slot = StorageData::from_value(B256::repeat_byte(4), value); + + // Clients verify range proofs against the RLP-encoded trie leaf, so the wire bytes must be + // exactly that rather than a fixed-width word. + assert_eq!(slot.data.as_ref(), alloy_rlp::encode(value)); + assert_eq!(slot.value().unwrap(), value); + } + + #[test] + fn storage_data_rejects_trailing_bytes() { + let mut slot = StorageData::from_value(B256::repeat_byte(4), U256::from(1)); + slot.data = [slot.data.as_ref(), &[0x00]].concat().into(); + + assert!(slot.value().is_err()); + } } diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index 51a31fc5396..fbf6d53b924 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -4,13 +4,10 @@ use crate::{ budget::DEFAULT_BUDGET_TRY_DRAIN_DOWNLOADERS, metered_poll_nested_stream_with_budget, metrics::EthRequestHandlerMetrics, }; -use alloy_consensus::{ - constants::{EMPTY_ROOT_HASH, KECCAK_EMPTY}, - BlockHeader, ReceiptWithBloom, -}; +use alloy_consensus::{constants::KECCAK_EMPTY, BlockHeader, ReceiptWithBloom}; use alloy_eips::BlockHashOrNumber; -use alloy_primitives::{Bytes, B256, U256}; -use alloy_rlp::{Encodable, RlpEncodable}; +use alloy_primitives::{Bytes, B256}; +use alloy_rlp::Encodable; use futures::StreamExt; use reth_eth_wire::{ snap::{ @@ -28,7 +25,7 @@ use reth_network_p2p::{ snap::client::SnapResponse, }; use reth_network_peers::PeerId; -use reth_primitives_traits::{Account, Block}; +use reth_primitives_traits::Block; use reth_storage_api::{ errors::provider::ProviderResult, BalProvider, BlockReader, BytecodeReader, GetBlockAccessListLimit, HeaderProvider, RangeEnd, RangeResponse, StateProviderFactory, @@ -553,8 +550,10 @@ where let mut account_data = Vec::with_capacity(accounts.len()); for (hash, account) in accounts { let storage_root = state.storage_root_by_hash(hash)?; - account_data - .push(AccountData { hash, body: slim_account_body(&account, storage_root) }); + account_data.push(AccountData::from_trie_account( + hash, + &account.into_trie_account(storage_root), + )); } Ok(AccountRangeMessage { request_id: req.request_id, accounts: account_data, proof }) @@ -610,11 +609,8 @@ where slots.push( account_slots .into_iter() - .map(|(hash, value)| StorageData { - hash, - // snap clients verify proofs against RLP-encoded storage trie leaves. - data: alloy_rlp::encode(value).into(), - }) + // snap clients verify proofs against RLP-encoded storage trie leaves. + .map(|(hash, value)| StorageData::from_value(hash, value)) .collect(), ); @@ -640,39 +636,6 @@ fn boundary_proof_keys(origin: B256, last: Option<&(B256, T)>) -> Vec { } } -/// Like the consensus trie account, but the code hash and storage root are empty byte strings -/// rather than [`KECCAK_EMPTY`]/[`EMPTY_ROOT_HASH`] when the account has no code/storage, to -/// avoid transferring the same 32 bytes for every EOA. Borrowed to encode without allocating. -#[derive(RlpEncodable)] -struct SlimAccountBody<'a> { - /// The account's nonce. - nonce: u64, - /// The account's balance. - balance: U256, - /// Empty when the account has no storage. - storage_root: &'a [u8], - /// Empty when the account has no code. - code_hash: &'a [u8], -} - -/// RLP-encodes `account` in snap/2's slim format; see [`SlimAccountBody`]. -fn slim_account_body(account: &Account, storage_root: B256) -> Bytes { - let storage_root: &[u8] = - if storage_root == EMPTY_ROOT_HASH { &[] } else { storage_root.as_slice() }; - let code_hash: &[u8] = match &account.bytecode_hash { - Some(hash) if *hash != KECCAK_EMPTY => hash.as_slice(), - _ => &[], - }; - - alloy_rlp::encode(SlimAccountBody { - nonce: account.nonce, - balance: account.balance, - storage_root, - code_hash, - }) - .into() -} - /// An endless future. /// /// This should be spawned or used as part of `tokio::select!`. @@ -850,12 +813,14 @@ pub enum IncomingEthRequest { #[cfg(test)] mod tests { use super::*; + use alloy_consensus::constants::EMPTY_ROOT_HASH; use alloy_eips::{ eip4844::{BlobAndProofV1, BlobAndProofV2, BlobCellsAndProofsV1}, eip7594::{BlobTransactionSidecarVariant, Cell}, }; - use alloy_primitives::{keccak256, Address, TxHash, B128}; + use alloy_primitives::{keccak256, Address, TxHash, B128, U256}; use reth_network_api::test_utils::PeersHandle; + use reth_primitives_traits::Account; use reth_provider::test_utils::{ExtendedAccount, MockEthProvider}; use reth_storage_api::noop::NoopProvider; use reth_transaction_pool::blobstore::{ diff --git a/crates/net/network/tests/it/snap/mod.rs b/crates/net/network/tests/it/snap/mod.rs index 811d9dfd4cc..cd78270e634 100644 --- a/crates/net/network/tests/it/snap/mod.rs +++ b/crates/net/network/tests/it/snap/mod.rs @@ -13,7 +13,6 @@ use alloy_eip7928::{ }; use alloy_eips::NumHash; use alloy_primitives::{keccak256, Address, Bytes, B256, U256}; -use alloy_rlp::{Decodable, RlpDecodable}; use alloy_trie::{nodes::RlpNode, proof::verify_proof, Nibbles}; use reth_chainspec::Hardforks; use reth_eth_wire::{ @@ -205,19 +204,6 @@ fn assert_boundary_proof(root: B256, key: B256, expected_value: Option>, ); } -/// Owned decode counterpart of `eth_requests::SlimAccountBody` -#[derive(Debug, RlpDecodable)] -struct SlimAccountBody { - /// The account's nonce. - nonce: u64, - /// The account's balance. - balance: U256, - /// Empty when the account has no storage. - storage_root: Bytes, - /// Empty when the account has no code. - code_hash: Bytes, -} - /// A valid RLP-encoded EIP-7928 block access list for `address`, with its commitment hash. fn valid_bal(address: Address) -> (Bytes, B256) { let mut change = AccountChanges::new(address); @@ -280,16 +266,16 @@ async fn account_range_roundtrip_carries_slim_encoding_and_proof() { }; assert_eq!(request_id, 7); assert_eq!(returned.len(), expected.len()); - for (AccountData { hash, body }, (expected_hash, expected_account)) in - returned.iter().zip(&expected) - { - assert_eq!(hash, expected_hash); - let decoded = SlimAccountBody::decode(&mut &body[..]).unwrap(); + for (account, (expected_hash, expected_account)) in returned.iter().zip(&expected) { + assert_eq!(&account.hash, expected_hash); + let decoded = account.trie_account().unwrap(); assert_eq!(decoded.nonce, expected_account.nonce); assert_eq!(decoded.balance, expected_account.balance); - // Freshly generated EOAs have no storage/code, so they get the slim (elided) encoding. - assert!(decoded.storage_root.is_empty()); - assert!(decoded.code_hash.is_empty()); + // Freshly generated EOAs have no storage/code, so the slim encoding elides both fields + // and they decode back to their defaults. + assert_eq!(decoded.storage_root, EMPTY_ROOT_HASH); + assert_eq!(decoded.code_hash, KECCAK_EMPTY); + assert_eq!(account, &AccountData::from_trie_account(*expected_hash, &decoded)); } assert!(!proof.is_empty()); @@ -422,10 +408,8 @@ async fn storage_range_roundtrip_carries_rlp_values_and_proof() { assert_eq!(request_id, 9); assert_eq!(returned.len(), 1); - let decoded: Vec<_> = returned[0] - .iter() - .map(|slot| (slot.hash, U256::decode(&mut &slot.data[..]).unwrap())) - .collect(); + let decoded: Vec<_> = + returned[0].iter().map(|slot| (slot.hash, slot.value().unwrap())).collect(); let expected_bounded: Vec<_> = expected.iter().filter(|(hash, _)| *hash >= origin && *hash <= limit).copied().collect(); assert_eq!(decoded, expected_bounded); @@ -493,10 +477,8 @@ async fn storage_range_empty_window_returns_boundary_slot() { panic!("expected a storage ranges response"); }; assert_eq!(returned.len(), 1); - let decoded: Vec<_> = returned[0] - .iter() - .map(|slot| (slot.hash, U256::decode(&mut &slot.data[..]).unwrap())) - .collect(); + let decoded: Vec<_> = + returned[0].iter().map(|slot| (slot.hash, slot.value().unwrap())).collect(); assert_eq!(decoded, vec![expected[2]]); assert!(!proof.is_empty()); @@ -573,15 +555,11 @@ async fn storage_ranges_multi_account_bounds_only_first_account() { panic!("expected a storage ranges response"); }; assert_eq!(returned.len(), 2, "both accounts should appear"); - let decoded_a: Vec<_> = returned[0] - .iter() - .map(|slot| (slot.hash, U256::decode(&mut &slot.data[..]).unwrap())) - .collect(); + let decoded_a: Vec<_> = + returned[0].iter().map(|slot| (slot.hash, slot.value().unwrap())).collect(); assert_eq!(decoded_a, expected_a, "the earlier account's range should be complete"); - let decoded_b: Vec<_> = returned[1] - .iter() - .map(|slot| (slot.hash, U256::decode(&mut &slot.data[..]).unwrap())) - .collect(); + let decoded_b: Vec<_> = + returned[1].iter().map(|slot| (slot.hash, slot.value().unwrap())).collect(); assert!(decoded_b.len() < expected_b.len(), "the final account's range should be truncated"); assert_eq!(decoded_b, expected_b[..decoded_b.len()]); assert!(!proof.is_empty()); @@ -610,10 +588,8 @@ async fn storage_ranges_multi_account_bounds_only_first_account() { panic!("expected a storage ranges response"); }; assert_eq!(returned.len(), 1, "only the bounded first account should appear"); - let decoded_a: Vec<_> = returned[0] - .iter() - .map(|slot| (slot.hash, U256::decode(&mut &slot.data[..]).unwrap())) - .collect(); + let decoded_a: Vec<_> = + returned[0].iter().map(|slot| (slot.hash, slot.value().unwrap())).collect(); assert_eq!(decoded_a, expected_a[1..]); assert!(!proof.is_empty()); assert_boundary_proof(storage_root_a, origin, Some(alloy_rlp::encode(expected_a[1].1)), &proof); diff --git a/crates/net/p2p/src/snap/client.rs b/crates/net/p2p/src/snap/client.rs index f805a4c6061..c48ee6a41e6 100644 --- a/crates/net/p2p/src/snap/client.rs +++ b/crates/net/p2p/src/snap/client.rs @@ -1,9 +1,17 @@ -use crate::{download::DownloadClient, error::PeerRequestResult, priority::Priority}; +use crate::{ + download::DownloadClient, + error::{PeerRequestResult, RequestError}, + full_block::NoopFullBlockClient, + priority::Priority, +}; use futures::Future; -use reth_eth_wire_types::snap::{ - AccountRangeMessage, BlockAccessListsMessage, ByteCodesMessage, GetAccountRangeMessage, - GetBlockAccessListsMessage, GetByteCodesMessage, GetStorageRangesMessage, SnapProtocolMessage, - StorageRangesMessage, +use reth_eth_wire_types::{ + snap::{ + AccountRangeMessage, BlockAccessListsMessage, ByteCodesMessage, GetAccountRangeMessage, + GetBlockAccessListsMessage, GetByteCodesMessage, GetStorageRangesMessage, + SnapProtocolMessage, StorageRangesMessage, + }, + NetworkPrimitives, }; /// Response types for snap sync requests @@ -110,6 +118,66 @@ pub trait SnapClient: DownloadClient { ) -> Self::Output; } +/// Fails every snap request with [`RequestError::UnsupportedCapability`], so the noop client can +/// stand in wherever a [`SnapClient`] bound is required but snap is not served. +impl SnapClient for NoopFullBlockClient +where + Net: NetworkPrimitives, +{ + type Output = futures::future::Ready>; + + /// Fails the account range request as unsupported. + fn get_account_range_with_priority( + &self, + _request: GetAccountRangeMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } + + /// Fails the storage ranges request as unsupported. + fn get_storage_ranges(&self, _request: GetStorageRangesMessage) -> Self::Output { + unsupported() + } + + /// Fails the prioritized storage ranges request as unsupported. + fn get_storage_ranges_with_priority( + &self, + _request: GetStorageRangesMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } + + /// Fails the bytecode request as unsupported. + fn get_byte_codes(&self, _request: GetByteCodesMessage) -> Self::Output { + unsupported() + } + + /// Fails the prioritized bytecode request as unsupported. + fn get_byte_codes_with_priority( + &self, + _request: GetByteCodesMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } + + /// Fails the block access lists request as unsupported. + fn get_block_access_lists_with_priority( + &self, + _request: GetBlockAccessListsMessage, + _priority: Priority, + ) -> Self::Output { + unsupported() + } +} + +/// The noop answer to any snap request: immediately ready, no capability. +fn unsupported() -> futures::future::Ready> { + futures::future::ready(Err(RequestError::UnsupportedCapability)) +} + #[cfg(test)] mod tests { use super::*; From 47ac8d02d9cadc1db62eeda639c28b3e86ad9378 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 5 Aug 2026 17:47:29 +0200 Subject: [PATCH 098/148] revert(provider): return header with transaction metadata (#26595) --- .../src/providers/blockchain_provider.rs | 40 ++----------- .../provider/src/providers/consistent.rs | 25 +------- .../provider/src/providers/database/mod.rs | 11 +--- .../src/providers/database/provider.rs | 57 ++++++++----------- .../storage/storage-api/src/transactions.rs | 31 +--------- 5 files changed, 33 insertions(+), 131 deletions(-) diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 6085fbc1dda..4782d6fded9 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -6,10 +6,10 @@ use crate::{ BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, CanonStateSubscriptions, ChainSpecProvider, ChainStateBlockReader, ChangeSetReader, - DatabaseProviderFactory, HeaderProvider, ProviderError, ProviderFactory, ProviderHeader, - PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, RocksDBProviderFactory, - StageCheckpointReader, StateProviderBox, StateProviderFactory, StateReader, - StaticFileProviderFactory, TransactionVariant, TransactionsProvider, + DatabaseProviderFactory, HeaderProvider, ProviderError, ProviderFactory, PruneCheckpointReader, + ReceiptProvider, ReceiptProviderIdExt, RocksDBProviderFactory, StageCheckpointReader, + StateProviderBox, StateProviderFactory, StateReader, StaticFileProviderFactory, + TransactionVariant, TransactionsProvider, }; use alloy_consensus::{transaction::TransactionMeta, BlockHeader}; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumHash, BlockNumberOrTag}; @@ -603,15 +603,6 @@ impl TransactionsProvider for BlockchainProvider { self.consistent_provider()?.transaction_by_hash_with_meta(tx_hash) } - fn transaction_by_hash_with_meta_and_header( - &self, - tx_hash: TxHash, - ) -> ProviderResult< - Option<(Self::Transaction, TransactionMeta, SealedHeader>)>, - > { - self.consistent_provider()?.transaction_by_hash_with_meta_and_header(tx_hash) - } - fn transactions_by_block( &self, id: BlockHashOrNumber, @@ -1011,7 +1002,7 @@ mod tests { }, BlockWriter, CanonChainTracker, ProviderFactory, SaveBlocksInput, }; - use alloy_consensus::{constants::EMPTY_ROOT_HASH, transaction::TransactionMeta, BlockHeader}; + use alloy_consensus::constants::EMPTY_ROOT_HASH; use alloy_eips::{BlockHashOrNumber, BlockNumHash, BlockNumberOrTag}; use alloy_primitives::{keccak256, Address, BlockNumber, TxNumber, B256, U256}; use itertools::Itertools; @@ -2679,27 +2670,6 @@ mod tests { ), B256::random() ), - ( - ONE, - transaction_by_hash_with_meta_and_header, - |block: &SealedBlock, _: TxNumber, tx_hash: B256, _: &Vec>| ( - tx_hash, - Some(( - block.body().transactions[test_tx_index].clone(), - TransactionMeta { - tx_hash, - index: test_tx_index as u64, - block_hash: block.hash(), - block_number: block.number, - base_fee: block.base_fee_per_gas(), - excess_blob_gas: block.excess_blob_gas(), - timestamp: block.timestamp(), - }, - block.clone_sealed_header(), - )) - ), - B256::random() - ), ( ONE, block_by_transaction_id, diff --git a/crates/storage/provider/src/providers/consistent.rs b/crates/storage/provider/src/providers/consistent.rs index 863b57ae534..d10ac824c21 100644 --- a/crates/storage/provider/src/providers/consistent.rs +++ b/crates/storage/provider/src/providers/consistent.rs @@ -2,7 +2,7 @@ use super::{DatabaseProviderRO, ProviderFactory, ProviderNodeTypes}; use crate::{ providers::{StaticFileProvider, StaticFileProviderRWRefMut}, to_range, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, - BlockSource, ChainSpecProvider, ChangeSetReader, HeaderProvider, ProviderError, ProviderHeader, + BlockSource, ChainSpecProvider, ChangeSetReader, HeaderProvider, ProviderError, PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateReader, StaticFileProviderFactory, TransactionVariant, TransactionsProvider, }; @@ -814,29 +814,6 @@ impl TransactionsProvider for ConsistentProvider { self.storage_provider.transaction_by_hash_with_meta(tx_hash) } - fn transaction_by_hash_with_meta_and_header( - &self, - tx_hash: TxHash, - ) -> ProviderResult< - Option<(Self::Transaction, TransactionMeta, SealedHeader>)>, - > { - if let Some((tx, meta, header)) = self.head_block.as_ref().and_then(|head| { - head.chain().find_map(|block_state| { - block_state.find_indexed(tx_hash).map(|indexed| { - ( - indexed.tx().clone(), - indexed.meta(), - block_state.block_ref().recovered_block().clone_sealed_header(), - ) - }) - }) - }) { - return Ok(Some((tx, meta, header))) - } - - self.storage_provider.transaction_by_hash_with_meta_and_header(tx_hash) - } - fn transactions_by_block( &self, id: BlockHashOrNumber, diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index fcb6b9ec069..fb2bb49b629 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -7,7 +7,7 @@ use crate::{ traits::{BlockSource, ReceiptProvider}, BalProvider, BalStoreHandle, BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, EitherWriterDestination, HeaderProvider, HeaderSyncGapProvider, - InMemoryBalStore, MetadataProvider, ProviderError, ProviderHeader, PruneCheckpointReader, + InMemoryBalStore, MetadataProvider, ProviderError, PruneCheckpointReader, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StaticFileProviderFactory, StaticFileWriter, TransactionVariant, TransactionsProvider, }; @@ -821,15 +821,6 @@ impl TransactionsProvider for ProviderFactory { self.provider()?.transaction_by_hash_with_meta(tx_hash) } - fn transaction_by_hash_with_meta_and_header( - &self, - tx_hash: TxHash, - ) -> ProviderResult< - Option<(Self::Transaction, TransactionMeta, SealedHeader>)>, - > { - self.provider()?.transaction_by_hash_with_meta_and_header(tx_hash) - } - fn transactions_by_block( &self, id: BlockHashOrNumber, diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 3dc41a9abdc..96d891e37e0 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -16,10 +16,10 @@ use crate::{ DBProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, HeaderProvider, HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, PersistenceFrontiers, - ProviderError, ProviderHeader, PruneCheckpointReader, PruneCheckpointWriter, RawRocksDBBatch, - RevertsInit, RocksBatchArg, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, - StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, - TransactionVariant, TransactionsProvider, TransactionsProviderExt, TrieWriter, + ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RawRocksDBBatch, RevertsInit, + RocksBatchArg, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateWriter, + StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, + TransactionsProvider, TransactionsProviderExt, TrieWriter, }; use alloy_consensus::{ transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, @@ -2107,40 +2107,31 @@ impl TransactionsProvider for Datab &self, tx_hash: TxHash, ) -> ProviderResult> { - Ok(self - .transaction_by_hash_with_meta_and_header(tx_hash)? - .map(|(transaction, meta, _)| (transaction, meta))) - } - - fn transaction_by_hash_with_meta_and_header( - &self, - tx_hash: TxHash, - ) -> ProviderResult< - Option<(Self::Transaction, TransactionMeta, SealedHeader>)>, - > { if let Some(transaction_id) = self.transaction_id(tx_hash)? && let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? && let Some(block_number) = self.block_by_transaction_id(transaction_id)? && - let Some(sealed_header) = self.sealed_header(block_number)? && - let Some(block_body) = self.block_body_indices(block_number)? + let Some(sealed_header) = self.sealed_header(block_number)? { - // the index of the tx in the block is the offset: - // len([start..tx_id]) - // NOTE: `transaction_id` is always `>=` the block's first - // index - let index = transaction_id - block_body.first_tx_num(); - - let meta = TransactionMeta { - tx_hash, - index, - block_hash: sealed_header.hash(), - block_number, - base_fee: sealed_header.base_fee_per_gas(), - excess_blob_gas: sealed_header.excess_blob_gas(), - timestamp: sealed_header.timestamp(), - }; + let (header, block_hash) = sealed_header.split(); + if let Some(block_body) = self.block_body_indices(block_number)? { + // the index of the tx in the block is the offset: + // len([start..tx_id]) + // NOTE: `transaction_id` is always `>=` the block's first + // index + let index = transaction_id - block_body.first_tx_num(); + + let meta = TransactionMeta { + tx_hash, + index, + block_hash, + block_number, + base_fee: header.base_fee_per_gas(), + excess_blob_gas: header.excess_blob_gas(), + timestamp: header.timestamp(), + }; - return Ok(Some((transaction, meta, sealed_header))) + return Ok(Some((transaction, meta))) + } } Ok(None) diff --git a/crates/storage/storage-api/src/transactions.rs b/crates/storage/storage-api/src/transactions.rs index 2b32465aadb..354504d9667 100644 --- a/crates/storage/storage-api/src/transactions.rs +++ b/crates/storage/storage-api/src/transactions.rs @@ -1,10 +1,10 @@ -use crate::{BlockNumReader, BlockReader, HeaderProvider}; +use crate::{BlockNumReader, BlockReader}; use alloc::vec::Vec; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::BlockHashOrNumber; use alloy_primitives::{Address, BlockNumber, TxHash, TxNumber}; use core::ops::{Range, RangeBounds, RangeInclusive}; -use reth_primitives_traits::{SealedHeader, SignedTransaction}; +use reth_primitives_traits::SignedTransaction; use reth_storage_errors::provider::{ProviderError, ProviderResult}; /// Enum to control transaction hash inclusion. @@ -49,33 +49,6 @@ pub trait TransactionsProvider: BlockNumReader + Send { hash: TxHash, ) -> ProviderResult>; - /// Get transaction by transaction hash, additional metadata, and the sealed block header. - /// - /// Implementers that already load the header while constructing [`TransactionMeta`] should - /// override this method to return it without another lookup. - #[auto_impl(keep_default_for(&, Arc))] - #[expect(clippy::type_complexity)] - fn transaction_by_hash_with_meta_and_header( - &self, - hash: TxHash, - ) -> ProviderResult< - Option<( - Self::Transaction, - TransactionMeta, - SealedHeader<::Header>, - )>, - > - where - Self: HeaderProvider, - { - let Some((transaction, meta)) = self.transaction_by_hash_with_meta(hash)? else { - return Ok(None) - }; - let Some(header) = self.sealed_header_by_hash(meta.block_hash)? else { return Ok(None) }; - - Ok(Some((transaction, meta, header))) - } - /// Get transactions by block id. fn transactions_by_block( &self, From a4e6baa9d2de15c54b2afbfb754d172a5e05fd67 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 5 Aug 2026 19:53:58 +0200 Subject: [PATCH 099/148] fix(rpc): restore optional receipt conversion (#26596) Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- crates/rpc/rpc-convert/src/transaction.rs | 31 +++++++++++++-- crates/rpc/rpc-eth-api/src/helpers/block.rs | 2 +- crates/rpc/rpc-eth-api/src/helpers/receipt.rs | 38 ++++++++----------- .../rpc-eth-api/src/helpers/subscriptions.rs | 2 +- crates/rpc/rpc-eth-types/src/block.rs | 2 +- crates/rpc/rpc-eth-types/src/receipt.rs | 7 ++-- 6 files changed, 49 insertions(+), 33 deletions(-) diff --git a/crates/rpc/rpc-convert/src/transaction.rs b/crates/rpc/rpc-convert/src/transaction.rs index 4b010776a2f..22d70029111 100644 --- a/crates/rpc/rpc-convert/src/transaction.rs +++ b/crates/rpc/rpc-convert/src/transaction.rs @@ -50,12 +50,21 @@ pub trait ReceiptConverter: Debug + 'static { header: &SealedHeaderFor, ) -> Result; - /// Converts primitive receipts from `block` to RPC representations. + /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all + /// receipts are from the same block. fn convert_receipts( &self, receipts: Vec>, - block: &SealedBlock, ) -> Result, Self::Error>; + + /// Converts primitive receipts from `block` to RPC representations. + fn convert_receipts_with_block( + &self, + receipts: Vec>, + _block: &SealedBlock, + ) -> Result, Self::Error> { + self.convert_receipts(receipts) + } } /// A type that knows how to convert a consensus header into an RPC header. @@ -170,10 +179,17 @@ pub trait RpcConvert: Send + Sync + Unpin + Debug + DynClone + 'static { header: &SealedHeaderFor, ) -> Result, Self::Error>; - /// Converts primitive receipts from `block` to RPC representations. + /// Converts a set of primitive receipts to RPC representations. It is guaranteed that all + /// receipts are from the same block. fn convert_receipts( &self, receipts: Vec>, + ) -> Result>, Self::Error>; + + /// Converts primitive receipts from `block` to RPC representations. + fn convert_receipts_with_block( + &self, + receipts: Vec>, block: &SealedBlock>, ) -> Result>, Self::Error>; @@ -746,9 +762,16 @@ where fn convert_receipts( &self, receipts: Vec>, + ) -> Result>, Self::Error> { + self.receipt_converter.convert_receipts(receipts) + } + + fn convert_receipts_with_block( + &self, + receipts: Vec>, block: &SealedBlock>, ) -> Result>, Self::Error> { - self.receipt_converter.convert_receipts(receipts, block) + self.receipt_converter.convert_receipts_with_block(receipts, block) } fn convert_header( diff --git a/crates/rpc/rpc-eth-api/src/helpers/block.rs b/crates/rpc/rpc-eth-api/src/helpers/block.rs index 0d1d4305897..479b45623c4 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/block.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/block.rs @@ -136,7 +136,7 @@ pub trait EthBlocks: LoadBlock impl Future, Self::Error>> + Send { async move { let hash = meta.block_hash; - let block = match block { - Some(block) => block, - None => self - .cache() - .get_recovered_block(hash) - .await - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(hash.into()))?, - }; // Use pre-fetched receipts if available, otherwise fetch from cache. let all_receipts = match all_receipts { Some(receipts) => receipts, @@ -55,20 +46,21 @@ pub trait LoadReceipt: let (gas_used, next_log_index) = calculate_gas_used_and_next_log_index(meta.index, &all_receipts); - Ok(self - .converter() - .convert_receipts( - vec![ConvertReceiptInput { - tx: tx.as_recovered_ref(), - gas_used: receipt.cumulative_gas_used() - gas_used, - receipt, - next_log_index, - meta, - }], - block.sealed_block(), - )? - .pop() - .unwrap()) + let inputs = vec![ConvertReceiptInput { + tx: tx.as_recovered_ref(), + gas_used: receipt.cumulative_gas_used() - gas_used, + receipt, + next_log_index, + meta, + }]; + let mut receipts = match block { + Some(block) => { + self.converter().convert_receipts_with_block(inputs, block.sealed_block()) + } + None => self.converter().convert_receipts(inputs), + }?; + + Ok(receipts.pop().unwrap()) } } } diff --git a/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs b/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs index 51b93d3b7e2..fa40a1f76da 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/subscriptions.rs @@ -139,7 +139,7 @@ pub trait EthSubscriptions: return None; } - match converter.convert_receipts(inputs, block.sealed_block()) { + match converter.convert_receipts_with_block(inputs, block.sealed_block()) { Ok(rpc_receipts) => Some(rpc_receipts), Err(err) => { error!(target = "rpc", %err, "Failed to convert receipts"); diff --git a/crates/rpc/rpc-eth-types/src/block.rs b/crates/rpc/rpc-eth-types/src/block.rs index e2f5cd3f94e..f47ecc0b534 100644 --- a/crates/rpc/rpc-eth-types/src/block.rs +++ b/crates/rpc/rpc-eth-types/src/block.rs @@ -183,7 +183,7 @@ where calculate_gas_used_and_next_log_index(meta.index, all_receipts); converter - .convert_receipts( + .convert_receipts_with_block( vec![ConvertReceiptInput { tx: tx.recovered_tx(), gas_used: receipt.cumulative_gas_used() - gas_used, diff --git a/crates/rpc/rpc-eth-types/src/receipt.rs b/crates/rpc/rpc-eth-types/src/receipt.rs index 6a711853c6c..b16234e927c 100644 --- a/crates/rpc/rpc-eth-types/src/receipt.rs +++ b/crates/rpc/rpc-eth-types/src/receipt.rs @@ -7,7 +7,7 @@ use alloy_primitives::{Address, TxKind}; use alloy_rpc_types_eth::{Log, TransactionReceipt}; use reth_chainspec::EthChainSpec; use reth_ethereum_primitives::Receipt; -use reth_primitives_traits::{NodePrimitives, SealedBlock, SealedHeaderFor, TransactionMeta}; +use reth_primitives_traits::{NodePrimitives, SealedHeaderFor, TransactionMeta}; use reth_rpc_convert::transaction::{ConvertReceiptInput, ReceiptConverter}; use std::sync::Arc; @@ -131,12 +131,13 @@ where fn convert_receipts( &self, inputs: Vec>, - _block: &SealedBlock, ) -> Result, Self::Error> { let mut receipts = Vec::with_capacity(inputs.len()); + let blob_params = inputs + .first() + .and_then(|input| self.chain_spec.blob_params_at_timestamp(input.meta.timestamp)); for input in inputs { - let blob_params = self.chain_spec.blob_params_at_timestamp(input.meta.timestamp); receipts.push(build_receipt(input, blob_params, &self.build_rpc_receipt)); } From b74a4f7eda2e4395a7a645eda16940512bac1aa3 Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:14:49 +0200 Subject: [PATCH 100/148] refactor(rpc): reuse optional receipt cache blocks (#26599) Co-authored-by: Matthias Seitz --- crates/rpc/rpc-eth-api/src/helpers/receipt.rs | 39 ++++++++++++++----- crates/rpc/rpc-eth-types/src/cache/mod.rs | 10 +++++ 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/receipt.rs b/crates/rpc/rpc-eth-api/src/helpers/receipt.rs index 201bd83ac32..2e9d6f3c896 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/receipt.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/receipt.rs @@ -21,7 +21,7 @@ pub trait LoadReceipt: { /// Helper method for `eth_getBlockReceipts` and `eth_getTransactionReceipt`. /// - /// If `all_receipts` is `Some`, skips the cache lookup for receipts entirely. + /// If a value is `Some`, skips the corresponding cache lookup entirely. fn build_transaction_receipt( &self, tx: Recovered>, @@ -32,15 +32,34 @@ pub trait LoadReceipt: ) -> impl Future, Self::Error>> + Send { async move { let hash = meta.block_hash; - // Use pre-fetched receipts if available, otherwise fetch from cache. - let all_receipts = match all_receipts { - Some(receipts) => receipts, - None => self - .cache() - .get_receipts(hash) - .await - .map_err(Self::Error::from_eth_err)? - .ok_or(EthApiError::HeaderNotFound(hash.into()))?, + let (block, all_receipts) = match (block, all_receipts) { + (Some(block), Some(all_receipts)) => (Some(block), all_receipts), + (Some(block), None) => { + let all_receipts = self + .cache() + .get_receipts(hash) + .await + .map_err(Self::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(hash.into()))?; + (Some(block), all_receipts) + } + (None, Some(all_receipts)) => { + let block = self + .cache() + .get_maybe_block(hash) + .await + .map_err(Self::Error::from_eth_err)?; + (block, all_receipts) + } + (None, None) => { + let (all_receipts, block) = self + .cache() + .get_receipts_and_maybe_block(hash) + .await + .map_err(Self::Error::from_eth_err)? + .ok_or(EthApiError::HeaderNotFound(hash.into()))?; + (block, all_receipts) + } }; let (gas_used, next_log_index) = diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index c2cc51bf659..40a186e234d 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -157,6 +157,16 @@ impl EthStateCache { rx.await.map_err(|_| CacheServiceUnavailable)? } + /// Requests the block for the given block hash if it is cached. + pub async fn get_maybe_block( + &self, + block_hash: B256, + ) -> ProviderResult>>> { + let (response_tx, rx) = oneshot::channel(); + let _ = self.to_service.send(CacheAction::GetCachedBlock { block_hash, response_tx }); + rx.await.map_err(|_| CacheServiceUnavailable.into()) + } + /// Requests the receipts for the block hash /// /// Returns `None` if the block was not found. From 1b20c9bfc17ca1fc0563683e26a79b2f29c33c3a Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 6 Aug 2026 00:11:08 +0200 Subject: [PATCH 101/148] fix(static-file): respect prune checkpoints in consistency check (#26565) Co-authored-by: 0xMars42 <195151467+0xMars42@users.noreply.github.com> --- crates/stages/stages/src/stages/mod.rs | 102 +++++++++++++++++- .../src/providers/static_file/manager.rs | 64 +++++++++-- .../src/providers/static_file/writer.rs | 8 +- .../src/providers/static_file/writer_tests.rs | 24 +++++ 4 files changed, 184 insertions(+), 14 deletions(-) diff --git a/crates/stages/stages/src/stages/mod.rs b/crates/stages/stages/src/stages/mod.rs index 1145af1b934..bd9daeeed34 100644 --- a/crates/stages/stages/src/stages/mod.rs +++ b/crates/stages/stages/src/stages/mod.rs @@ -53,6 +53,7 @@ mod tests { use reth_db::mdbx::{cursor::Cursor, RW}; use reth_db_api::{ cursor::{DbCursorRO, DbCursorRW}, + models::StorageSettings, table::Table, tables, transaction::{DbTx, DbTxMut}, @@ -67,14 +68,15 @@ mod tests { providers::{StaticFileProvider, StaticFileWriter}, test_utils::MockNodeTypesWithDB, AccountExtReader, BlockBodyIndicesProvider, BlockWriter, DatabaseProviderFactory, - ProviderFactory, ProviderResult, ReceiptProvider, StageCheckpointWriter, - StaticFileProviderFactory, StorageReader, + ProviderFactory, ProviderResult, PruneCheckpointWriter, ReceiptProvider, + StageCheckpointWriter, StaticFileProviderFactory, StorageReader, }; - use reth_prune_types::{PruneMode, PruneModes}; + use reth_prune_types::{PruneCheckpoint, PruneMode, PruneModes, PruneSegment}; use reth_stages_api::{ ExecInput, ExecutionStageThresholds, PipelineTarget, Stage, StageCheckpoint, StageId, }; use reth_static_file_types::StaticFileSegment; + use reth_storage_api::StorageSettingsCache; use reth_testing_utils::generators::{ self, random_block, random_block_range, random_receipt, BlockRangeParams, }; @@ -293,6 +295,42 @@ mod tests { Ok(db) } + fn seed_v2_data() -> TestStageDB { + let db = seed_data(90).unwrap(); + db.factory.set_storage_settings_cache(StorageSettings::v2()); + db + } + + fn save_prune_checkpoints( + db: &TestStageDB, + checkpoints: impl IntoIterator, + ) { + let provider_rw = db.factory.provider_rw().unwrap(); + for (segment, block_number, prune_mode) in checkpoints { + provider_rw + .save_prune_checkpoint( + segment, + PruneCheckpoint { + block_number: Some(block_number), + tx_number: None, + prune_mode, + }, + ) + .unwrap(); + } + provider_rw.commit().unwrap(); + } + + fn assert_consistency(db: &TestStageDB, expected: Option) { + assert_eq!( + db.factory + .static_file_provider() + .check_consistency(&db.factory.database_provider_ro().unwrap()) + .unwrap(), + expected + ); + } + /// Simulates losing data to corruption and compare the check consistency result /// against the expected one. fn simulate_behind_checkpoint_corruption( @@ -534,4 +572,62 @@ mod tests { // Fill the gap, and ensure no unwind is necessary. update_db_and_check::(&db, current + 1, None); } + + #[test] + fn test_consistency_pruned_v2_segments() { + for prune_mode in [PruneMode::Distance(2_000_000), PruneMode::Before(90)] { + let db = seed_v2_data(); + + // Without prune checkpoints the missing v2 segments look like data loss. + assert_consistency(&db, Some(PipelineTarget::Unwind(0))); + + save_prune_checkpoints( + &db, + [ + PruneSegment::SenderRecovery, + PruneSegment::AccountHistory, + PruneSegment::StorageHistory, + ] + .map(|segment| (segment, 89, prune_mode)), + ); + assert_consistency(&db, None); + } + } + + #[test] + fn test_consistency_unwind_bounded_by_prune_checkpoint() { + let db = seed_v2_data(); + + // Senders are only pruned up to block 50 while the stage checkpoint is at the tip: + // blocks 51..=89 did lose data, so an unwind is still required, but bounded by the + // prune checkpoint instead of going all the way to 0. + save_prune_checkpoints( + &db, + [ + (PruneSegment::SenderRecovery, 50, PruneMode::Distance(39)), + (PruneSegment::AccountHistory, 89, PruneMode::Distance(2_000_000)), + (PruneSegment::StorageHistory, 89, PruneMode::Distance(2_000_000)), + ], + ); + assert_consistency(&db, Some(PipelineTarget::Unwind(50))); + } + + #[test] + fn test_consistency_receipts_distance_prune_checkpoint() { + let db = seed_data(90).unwrap(); + let static_file_provider = db.factory.static_file_provider(); + + // Remove all receipts static files, like a node whose receipts have been + // distance-pruned away. + while let Some(block) = + static_file_provider.get_highest_static_file_block(StaticFileSegment::Receipts) + { + static_file_provider.delete_jar(StaticFileSegment::Receipts, block).unwrap(); + } + + assert_consistency(&db, Some(PipelineTarget::Unwind(0))); + + save_prune_checkpoints(&db, [(PruneSegment::Receipts, 89, PruneMode::Distance(2_000_000))]); + assert_consistency(&db, None); + } } diff --git a/crates/storage/provider/src/providers/static_file/manager.rs b/crates/storage/provider/src/providers/static_file/manager.rs index 6f8ef1c916a..0213cd96fb0 100644 --- a/crates/storage/provider/src/providers/static_file/manager.rs +++ b/crates/storage/provider/src/providers/static_file/manager.rs @@ -1573,7 +1573,7 @@ impl StaticFileProvider { highest_block: Option, ) -> ProviderResult> where - Provider: DBProvider + BlockReader + StageCheckpointReader, + Provider: DBProvider + BlockReader + StageCheckpointReader + PruneCheckpointReader, N: NodePrimitives, { match segment { @@ -1645,7 +1645,7 @@ impl StaticFileProvider { highest_static_file_block: Option, ) -> ProviderResult> where - Provider: DBProvider + BlockReader + StageCheckpointReader, + Provider: DBProvider + BlockReader + StageCheckpointReader + PruneCheckpointReader, { debug!(target: "reth::providers::static_file", "Ensuring invariants"); let mut db_cursor = provider.tx_ref().cursor_read::()?; @@ -1691,15 +1691,18 @@ impl StaticFileProvider { provider.get_stage_checkpoint(stage_id)?.unwrap_or_default().block_number; debug!(target: "reth::providers::static_file", ?stage_id, checkpoint_block_number, "Retrieved stage checkpoint"); + let effective_coverage_block = + Self::effective_coverage_block(provider, segment, highest_static_file_block)?; + // If the checkpoint is ahead, then we lost static file data. May be data corruption. - if checkpoint_block_number > highest_static_file_block { + if checkpoint_block_number > effective_coverage_block { info!( target: "reth::providers::static_file", checkpoint_block_number, - unwind_target = highest_static_file_block, + unwind_target = effective_coverage_block, "Setting unwind target." ); - return Ok(Some(highest_static_file_block)); + return Ok(Some(effective_coverage_block)); } // If the checkpoint is ahead, or matches, then nothing to do. @@ -1779,7 +1782,7 @@ impl StaticFileProvider { block_from_key: F, ) -> ProviderResult> where - Provider: DBProvider + BlockReader + StageCheckpointReader, + Provider: DBProvider + BlockReader + StageCheckpointReader + PruneCheckpointReader, T: Table, F: Fn(&T::Key) -> BlockNumber, { @@ -1828,15 +1831,18 @@ impl StaticFileProvider { let checkpoint_block_number = provider.get_stage_checkpoint(stage_id)?.unwrap_or_default().block_number; - if checkpoint_block_number > highest_static_file_block { + let effective_coverage_block = + Self::effective_coverage_block(provider, segment, highest_static_file_block)?; + + if checkpoint_block_number > effective_coverage_block { info!( target: "reth::providers::static_file", checkpoint_block_number, - unwind_target = highest_static_file_block, + unwind_target = effective_coverage_block, ?segment, "Setting unwind target." ); - return Ok(Some(highest_static_file_block)) + return Ok(Some(effective_coverage_block)) } if checkpoint_block_number < highest_static_file_block { @@ -1863,6 +1869,46 @@ impl StaticFileProvider { Ok(None) } + /// Returns the highest block accounted for in this segment, either through data + /// present in static files or through data intentionally removed by pruning. + /// + /// Data below a segment's prune checkpoint has been intentionally deleted, so its + /// absence from static files is not an inconsistency. Without this, a pruned segment + /// whose stage checkpoint is ahead of its (empty) static files is treated as data + /// corruption and triggers an unwind to block 0, which aborts the node on startup. + /// See . + fn effective_coverage_block( + provider: &Provider, + segment: StaticFileSegment, + highest_static_file_block: BlockNumber, + ) -> ProviderResult + where + Provider: PruneCheckpointReader, + { + let Some(prune_segment) = Self::prune_segment_for_static_file(segment) else { + return Ok(highest_static_file_block) + }; + + let prune_checkpoint_block = provider + .get_prune_checkpoint(prune_segment)? + .and_then(|checkpoint| checkpoint.block_number) + .unwrap_or_default(); + + Ok(highest_static_file_block.max(prune_checkpoint_block)) + } + + /// Returns the prune segment that governs data availability for a static file segment, + /// or `None` if the segment is never pruned. + const fn prune_segment_for_static_file(segment: StaticFileSegment) -> Option { + match segment { + StaticFileSegment::Receipts => Some(PruneSegment::Receipts), + StaticFileSegment::TransactionSenders => Some(PruneSegment::SenderRecovery), + StaticFileSegment::AccountChangeSets => Some(PruneSegment::AccountHistory), + StaticFileSegment::StorageChangeSets => Some(PruneSegment::StorageHistory), + StaticFileSegment::Headers | StaticFileSegment::Transactions => None, + } + } + /// Returns the earliest available block number that has not been expired and is still /// available. /// diff --git a/crates/storage/provider/src/providers/static_file/writer.rs b/crates/storage/provider/src/providers/static_file/writer.rs index 7a0196714e2..0d33110d4f9 100644 --- a/crates/storage/provider/src/providers/static_file/writer.rs +++ b/crates/storage/provider/src/providers/static_file/writer.rs @@ -748,8 +748,12 @@ impl StaticFileProviderRW { let current_block = if let Some(current_block_number) = self.current_block_number() { current_block_number } else { - self.increment_block(0)?; - 0 + // A fresh file does not necessarily start at block 0: a writer opened for a + // pruned segment may be positioned on a later fixed range, in which case its + // first block is the expected start of that range. + let first_block = self.writer.user_header().expected_block_start(); + self.increment_block(first_block)?; + first_block }; match current_block.cmp(&advance_to) { diff --git a/crates/storage/provider/src/providers/static_file/writer_tests.rs b/crates/storage/provider/src/providers/static_file/writer_tests.rs index 99f012ef268..c8c0d645f46 100644 --- a/crates/storage/provider/src/providers/static_file/writer_tests.rs +++ b/crates/storage/provider/src/providers/static_file/writer_tests.rs @@ -883,4 +883,28 @@ mod tests { drop(writer); } + + /// `ensure_at_block` on a fresh file that does not start at block 0 (e.g. created for a + /// distance-pruned segment) must initialize the writer at the file's expected block + /// start instead of erroring out. + /// + /// See . + #[test] + fn test_ensure_at_block_on_empty_non_genesis_file() { + let static_dir = tempfile::tempdir().unwrap(); + let provider = setup_test_provider(&static_dir, 100); + + // A writer for block 250 opens a fresh empty file covering blocks 200..=299. + let mut writer = provider.get_writer(250, StaticFileSegment::TransactionSenders).unwrap(); + assert_eq!(writer.current_block_number(), None); + + writer.ensure_at_block(250).unwrap(); + assert_eq!(writer.current_block_number(), Some(250)); + writer.commit().unwrap(); + + assert_eq!( + provider.get_highest_static_file_block(StaticFileSegment::TransactionSenders), + Some(250) + ); + } } From 995c55a9d50acea58bd2933df5d4fa82c670acd1 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Thu, 6 Aug 2026 01:24:58 +0300 Subject: [PATCH 102/148] refactor(`eth_createAccessList`): unify env preparation (#26598) --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 67 ++++++---------------- 1 file changed, 19 insertions(+), 48 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 58431b19d39..70beba2634e 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -454,7 +454,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA /// [`BlockId`]. fn create_access_list_with( &self, - mut evm_env: EvmEnvFor, + evm_env: EvmEnvFor, at: BlockId, request: RpcTxReq<::Network>, state_override: Option, @@ -462,53 +462,23 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA where Self: Trace, { - self.spawn_blocking_io_fut(async move |this| { - let state = this.state_at_block_id(at).await?; - let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build(); - - if let Some(state_overrides) = state_override { - apply_state_overrides(state_overrides, &mut db) - .map_err(Self::Error::from_eth_err)?; - } - - // Read fields from request before consuming it in create_txn_env - let request_has_gas_limit = request.as_ref().gas_limit().is_some(); + self.spawn_with_state_at_block(at, |this, mut db| { let initial = request.as_ref().access_list().cloned().unwrap_or_default(); - - let mut tx_env = this.create_txn_env(&evm_env, request, &mut db)?; - - // we want to disable this in eth_createAccessList, since this is common practice used - // by other node impls and providers - evm_env.cfg_env.disable_block_gas_limit = true; - - // The basefee should be ignored for eth_createAccessList - // See: - // - evm_env.cfg_env.disable_base_fee = true; - - // Disabled because eth_createAccessList is sometimes used with non-eoa senders - evm_env.cfg_env.disable_eip3607 = true; - - // Disable additional fee charges (e.g. L2 operator fees), - // consistent with prepare_call_env and estimate_gas_with. - evm_env.cfg_env.disable_fee_charge = true; - - // Disable EIP-7825 transaction gas limit cap so that the gas limit - // fallback (block gas limit) is not rejected when it exceeds the - // per-tx cap (2^24 ≈ 16.7M post-Osaka). - evm_env.cfg_env.tx_gas_limit_cap = Some(u64::MAX); - - if !request_has_gas_limit && tx_env.gas_price() > 0 { - let cap = this.caller_gas_allowance(&mut db, &evm_env, &tx_env)?; - // no gas limit was provided in the request, so we need to cap the request's gas - // limit - tx_env.set_gas_limit(cap.min(evm_env.block_env.gas_limit())); - } - - let mut inspector = AccessListInspector::new(initial); - - let result = this.inspect(&mut db, evm_env.clone(), tx_env.clone(), &mut inspector)?; - let access_list = inspector.into_access_list(); + let (evm_env, mut tx_env) = this.prepare_call_env( + evm_env, + request, + &mut db, + EvmOverrides::state(state_override), + )?; + + let mut evm = this.evm_config().evm_with_env_and_inspector( + &mut db, + evm_env, + AccessListInspector::new(initial), + ); + + let result = evm.transact(tx_env.clone())?; + let access_list = core::mem::take(evm.inspector_mut()).into_access_list(); let gas_used = result.result.tx_gas_used(); tx_env.set_access_list(access_list.clone()); if let Err(err) = Self::Error::ensure_success(result.result) { @@ -520,7 +490,8 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA } // transact again to get the exact gas used - let result = this.transact(&mut db, evm_env, tx_env)?; + evm.disable_inspector(); + let result = evm.transact(tx_env)?; let gas_used = result.result.tx_gas_used(); let error = Self::Error::ensure_success(result.result).err().map(|e| e.to_string()); From db183628a981c805e8d79c84133223472f6b9765 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Wed, 5 Aug 2026 18:00:29 -0500 Subject: [PATCH 103/148] fix(dns): rejoin TXT character-strings for EIP-1459 entries over 255 bytes (#26602) --- crates/net/dns/src/resolver.rs | 120 ++++++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 3 deletions(-) diff --git a/crates/net/dns/src/resolver.rs b/crates/net/dns/src/resolver.rs index 7b966184bd6..3deff398a6b 100644 --- a/crates/net/dns/src/resolver.rs +++ b/crates/net/dns/src/resolver.rs @@ -2,7 +2,10 @@ use dashmap::DashMap; pub use hickory_resolver::{net::NetError, TokioResolver}; -use hickory_resolver::{proto::rr::RData, ConnectionProvider}; +use hickory_resolver::{ + proto::rr::{rdata::TXT, RData}, + ConnectionProvider, +}; use std::future::Future; use tracing::trace; @@ -27,13 +30,23 @@ impl Resolver for hickory_resolver::Resolver

{ RData::TXT(txt) => Some(txt), _ => None, })?; - let entry = txt.txt_data.first()?; - String::from_utf8(entry.to_vec()).ok() + txt_entry(txt) } } } } +/// Joins all ``s of a TXT record into a single entry. +/// +/// [RFC 1035](https://www.rfc-editor.org/rfc/rfc1035#section-3.3) limits a single +/// `` to 255 bytes, while an +/// [EIP-1459](https://eips.ethereum.org/EIPS/eip-1459) entry is only bounded by the 512 byte DNS +/// UDP limit. Entries above 255 bytes are therefore published as several ``s +/// which have to be rejoined without a separator to recover the entry. +fn txt_entry(txt: &TXT) -> Option { + String::from_utf8(txt.txt_data.concat()).ok() +} + /// An asynchronous DNS resolver /// /// See also [`TokioResolver`] @@ -114,3 +127,104 @@ impl Resolver for TimeoutResolver { None } } + +#[cfg(test)] +mod tests { + use super::*; + use alloy_primitives::keccak256; + use data_encoding::BASE32_NOPAD; + use hickory_resolver::{ + config::{ConnectionConfig, NameServerConfig, ResolverConfig}, + net::runtime::TokioRuntimeProvider, + proto::{ + op::{Message, OpCode}, + rr::Record, + serialize::binary::{BinDecodable, BinEncodable}, + }, + }; + use std::net::{Ipv4Addr, SocketAddr}; + use tokio::net::UdpSocket; + + /// Maximum size of a single RFC 1035 ``. + const MAX_CHARACTER_STRING: usize = 255; + + /// A branch entry of the size go-ethereum's writer emits, which exceeds what a single + /// `` can hold. + fn long_branch_entry() -> String { + let children = (0u8..13) + .map(|i| BASE32_NOPAD.encode(&keccak256([i]).as_slice()[..16])) + .collect::>() + .join(","); + format!("enrtree-branch:{children}") + } + + fn character_strings(entry: &str) -> Vec { + entry + .as_bytes() + .chunks(MAX_CHARACTER_STRING) + .map(|chunk| String::from_utf8(chunk.to_vec()).unwrap()) + .collect() + } + + /// Answers every query with a single TXT record made up of `strings`. + async fn spawn_txt_server(strings: Vec) -> SocketAddr { + let socket = UdpSocket::bind((Ipv4Addr::LOCALHOST, 0)).await.unwrap(); + let addr = socket.local_addr().unwrap(); + tokio::spawn(async move { + let mut buf = [0u8; 512]; + while let Ok((len, from)) = socket.recv_from(&mut buf).await { + let request = Message::from_bytes(&buf[..len]).unwrap(); + let query = request.queries.first().unwrap().clone(); + + let mut response = Message::response(request.metadata.id, OpCode::Query); + response.metadata.authoritative = true; + response.metadata.recursion_desired = request.metadata.recursion_desired; + response.metadata.recursion_available = true; + response.answers.push(Record::from_rdata( + query.name().clone(), + 60, + RData::TXT(TXT::new(strings.clone())), + )); + response.queries.push(query); + + socket.send_to(&response.to_bytes().unwrap(), from).await.unwrap(); + } + }); + addr + } + + fn resolver_for(addr: SocketAddr) -> DnsResolver { + let mut connection = ConnectionConfig::udp(); + connection.port = addr.port(); + let config = ResolverConfig::from_parts( + None, + vec![], + vec![NameServerConfig::new(addr.ip(), true, vec![connection])], + ); + DnsResolver::new( + TokioResolver::builder_with_config(config, TokioRuntimeProvider::default()) + .build() + .unwrap(), + ) + } + + #[test] + fn txt_entry_joins_character_strings() { + let entry = long_branch_entry(); + let strings = character_strings(&entry); + assert!(entry.len() > MAX_CHARACTER_STRING); + assert!(strings.len() > 1); + + assert_eq!(txt_entry(&TXT::new(strings)), Some(entry)); + } + + #[tokio::test] + async fn lookup_txt_reads_record_split_over_character_strings() { + let entry = long_branch_entry(); + let addr = spawn_txt_server(character_strings(&entry)).await; + + let resolved = resolver_for(addr).lookup_txt("YNEGZIWHOM7TOOSUATAPTM.example.org").await; + + assert_eq!(resolved, Some(entry)); + } +} From ff6e072b7f8bae1b60fcafaf96176a16497b9a3a Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 6 Aug 2026 13:25:03 +0200 Subject: [PATCH 104/148] perf(engine): centralize overlay anchoring logic (#26580) Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com> --- crates/engine/tree/src/tree/mod.rs | 94 +-- .../tree/src/tree/payload_processor/mod.rs | 21 +- .../src/tree/payload_processor/prewarm.rs | 20 +- .../engine/tree/src/tree/payload_validator.rs | 55 +- .../tree/src/tree/state_root_strategy/mod.rs | 55 +- .../tree/src/tree/txpool_prewarm/mod.rs | 13 +- .../tree/src/tree/txpool_prewarm/worker.rs | 22 +- .../src/providers/blockchain_provider.rs | 21 +- .../storage/provider/src/test_utils/mock.rs | 55 +- crates/storage/provider/src/traits/full.rs | 13 +- crates/storage/storage-api/src/noop.rs | 13 +- crates/storage/storage-overlay/src/builder.rs | 601 ++++++++++-------- crates/storage/storage-overlay/src/manager.rs | 85 +-- 13 files changed, 602 insertions(+), 466 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index 4b6ee75aa3b..f0bb957fb3f 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -29,10 +29,11 @@ use reth_primitives_traits::{ FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader, }; use reth_provider::{ - BalProvider, BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader, - DatabaseProviderFactory, ProviderError, SaveBlocksInput, StageCheckpointReader, - StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, - StorageSettingsCache, TransactionVariant, + BalProvider, BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, + ChangeSetReader, DatabaseProviderFactory, LatestStateProvider, ProviderError, + PruneCheckpointReader, SaveBlocksInput, StageCheckpointReader, StateProviderBox, + StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, + TransactionVariant, TryIntoHistoricalStateProvider, }; use reth_revm::database::StateProviderDatabase; use reth_stages_api::ControlFlow; @@ -116,35 +117,59 @@ const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64; pub struct StateProviderBuilder { /// The provider factory used to create providers. provider_factory: P, - /// The historical block hash to fetch state from. - historical: B256, - /// The blocks that form the chain from historical to target and are in memory. - overlay: Option>>, + /// Hash of the block whose state to provide. + parent_hash: B256, + /// Tracks the in-memory parent chain and its overlays. + overlay_manager: OverlayManager, } impl StateProviderBuilder { - /// Creates a new state provider from the provider factory, historical block hash and optional - /// overlaid blocks. + /// Creates a new state provider builder for `parent_hash`. pub const fn new( provider_factory: P, - historical: B256, - overlay: Option>>, + parent_hash: B256, + overlay_manager: OverlayManager, ) -> Self { - Self { provider_factory, historical, overlay } + Self { provider_factory, parent_hash, overlay_manager } } } impl StateProviderBuilder where - P: BlockReader + StateProviderFactory + StateReader + Clone, + P: DatabaseProviderFactory, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, { /// Creates a new state provider from this builder. pub fn build(&self) -> ProviderResult { - let mut provider = self.provider_factory.state_by_block_hash(self.historical)?; - if let Some(overlay) = self.overlay.clone() { - provider = Box::new(MemoryOverlayStateProvider::new(provider, overlay)) - } - Ok(provider) + let overlay_builder = self.overlay_manager.overlay_builder(self.parent_hash); + let provider = self.provider_factory.database_provider_ro()?; + let anchor = overlay_builder.anchor_at_parent(&provider)?; + let (provider, overlay): (StateProviderBox, _) = match anchor { + reth_storage_overlay::AnchorForParent::NoReverts { anchor, overlay } => { + debug!( + target: "engine::tree", + parent_hash = %self.parent_hash, + ?anchor, + "creating state provider from latest state" + ); + (Box::new(LatestStateProvider::new(provider)), overlay) + } + reth_storage_overlay::AnchorForParent::RevertsRequired { anchor, overlay, .. } => { + debug!( + target: "engine::tree", + parent_hash = %self.parent_hash, + ?anchor, + "creating state provider from historical state" + ); + (provider.try_into_history_at_block(anchor.number)?, overlay) + } + }; + Ok(Box::new(MemoryOverlayStateProvider::new(provider, overlay))) } } @@ -413,10 +438,13 @@ where + Clone + 'static, P::Provider: BlockReader + + PruneCheckpointReader + StageCheckpointReader + ChangeSetReader + StorageChangeSetReader - + StorageSettingsCache, + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, C: ConfigureEvm + 'static, T: PayloadTypes>, V: EngineValidator + WaitForCaches, @@ -3548,26 +3576,16 @@ where where P: BlockReader + StateProviderFactory + StateReader + Clone, { - if let Some((historical, blocks)) = self.state.tree_state.blocks_by_hash(hash) { - debug!(target: "engine::tree", %hash, %historical, "found canonical state for block in memory, creating provider builder"); - // the block leads back to the canonical chain - return Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - historical, - Some(blocks), - ))) - } - - // Check if the block is persisted - if let Some(header) = self.provider.header(hash)? { - debug!(target: "engine::tree", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); - // For persisted blocks, we create a builder that will fetch state directly from the - // database - return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None))) + if !self.state.tree_state.contains_hash(&hash) && self.provider.header(hash)?.is_none() { + debug!(target: "engine::tree", %hash, "no canonical state found for block"); + return Ok(None) } - debug!(target: "engine::tree", %hash, "no canonical state found for block"); - Ok(None) + Ok(Some(StateProviderBuilder::new( + self.provider.clone(), + hash, + self.state.tree_state.overlay_manager.clone(), + ))) } } diff --git a/crates/engine/tree/src/tree/payload_processor/mod.rs b/crates/engine/tree/src/tree/payload_processor/mod.rs index b5d8942903f..628155f3ca5 100644 --- a/crates/engine/tree/src/tree/payload_processor/mod.rs +++ b/crates/engine/tree/src/tree/payload_processor/mod.rs @@ -17,7 +17,10 @@ use reth_evm::{ ConfigureEvm, ConvertTx, ExecutableTxIterator, ExecutableTxTuple, SpecFor, TxEnvFor, }; use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; -use reth_provider::{BlockExecutionOutput, BlockReader, StateProviderFactory, StateReader}; +use reth_provider::{ + BlockExecutionOutput, BlockNumReader, DatabaseProviderFactory, PruneCheckpointReader, + StageCheckpointReader, StorageSettingsCache, TryIntoHistoricalStateProvider, +}; use reth_revm::db::BundleState; use reth_tasks::Runtime; pub use reth_trie_parallel::{ @@ -172,7 +175,13 @@ where parallel_bal_execution: bool, ) -> IteratorPayloadHandle where - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory + Clone + 'static, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, { let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions, env.transaction_count, parallel_bal_execution); @@ -336,7 +345,13 @@ where parallel_bal_execution: bool, ) -> CacheTaskHandle<::Receipt> where - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory + Clone + 'static, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, { // Each mode carries the capability its producers use; the rest is dropped here, so // unused capabilities do not keep the state-root task's update channel open. diff --git a/crates/engine/tree/src/tree/payload_processor/prewarm.rs b/crates/engine/tree/src/tree/payload_processor/prewarm.rs index 502dd0e0184..9de0c0a250f 100644 --- a/crates/engine/tree/src/tree/payload_processor/prewarm.rs +++ b/crates/engine/tree/src/tree/payload_processor/prewarm.rs @@ -27,7 +27,9 @@ use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, use reth_metrics::Metrics; use reth_primitives_traits::{Account, FastInstant as Instant, NodePrimitives}; use reth_provider::{ - AccountReader, BlockExecutionOutput, BlockReader, StateProviderFactory, StateReader, + AccountReader, BlockExecutionOutput, BlockNumReader, DatabaseProviderFactory, + PruneCheckpointReader, StageCheckpointReader, StorageSettingsCache, + TryIntoHistoricalStateProvider, }; use reth_revm::database::StateProviderDatabase; use reth_tasks::{pool::WorkerPool, Runtime}; @@ -90,7 +92,13 @@ where impl PrewarmCacheTask where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory + Clone + 'static, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, Evm: ConfigureEvm + 'static, { /// Initializes the task with the given transactions pending execution @@ -561,7 +569,13 @@ type PrewarmEvmState = impl PrewarmContext where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + 'static, + P: DatabaseProviderFactory, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, Evm: ConfigureEvm + 'static, { /// Creates a per-thread EVM for prewarming. diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index 28b6e29e189..b67ba7b2ac6 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -146,10 +146,10 @@ use reth_primitives_traits::{ RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, }; use reth_provider::{ - providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader, - ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, ProviderError, - PruneCheckpointReader, StageCheckpointReader, StateProvider, StateProviderBox, - StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache, + providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader, ChangeSetReader, + DatabaseProviderFactory, DatabaseProviderROFactory, ProviderError, PruneCheckpointReader, + StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader, + StorageChangeSetReader, StorageSettingsCache, TryIntoHistoricalStateProvider, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; use reth_storage_overlay::OverlayManager; @@ -307,21 +307,17 @@ where + PruneCheckpointReader + ChangeSetReader + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, > + BlockReader

+ ChangeSetReader - + BlockNumReader + StateProviderFactory + StateReader + Clone - + Send - + Sync + 'static, OverlayStateProviderFactory: DatabaseProviderROFactory + Clone - + Send - + Sync + 'static, Evm: ConfigureEvm + 'static, { @@ -1396,33 +1392,22 @@ where /// Creates a `StateProviderBuilder` for the given parent hash. /// - /// This method checks if the parent is in the tree state (in-memory) or persisted to disk, - /// and creates the appropriate provider builder. + /// Returns `None` when the parent is neither in memory nor persisted. fn state_provider_builder( &self, hash: B256, state: &EngineApiTreeState, ) -> ProviderResult>> { - if let Some((historical, blocks)) = state.tree_state.blocks_by_hash(hash) { - debug!(target: "engine::tree::payload_validator", %hash, %historical, "found canonical state for block in memory, creating provider builder"); - // the block leads back to the canonical chain - return Ok(Some(StateProviderBuilder::new( - self.provider.clone(), - historical, - Some(blocks), - ))) + if !state.tree_state.contains_hash(&hash) && self.provider.header(hash)?.is_none() { + debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block"); + return Ok(None) } - // Check if the block is persisted - if let Some(header) = self.provider.header(hash)? { - debug!(target: "engine::tree::payload_validator", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder"); - // For persisted blocks, we create a builder that will fetch state directly from the - // database - return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None))) - } - - debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block"); - Ok(None) + Ok(Some(StateProviderBuilder::new( + self.provider.clone(), + hash, + state.tree_state.overlay_manager.clone(), + ))) } /// Called when an invalid block is encountered during validation. @@ -1793,19 +1778,17 @@ where + PruneCheckpointReader + ChangeSetReader + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, > + BlockReader
+ StateProviderFactory + StateReader + ChangeSetReader - + BlockNumReader + Clone + 'static, OverlayStateProviderFactory: DatabaseProviderROFactory + Clone - + Send - + Sync + 'static, N: NodePrimitives, V: PayloadValidator + Clone, diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index b2d84453352..83b3296a4ee 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -69,9 +69,10 @@ use reth_primitives_traits::{ AlloyBlockHeader, FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedHeader, }; use reth_provider::{ - providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader, + providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, - StateProviderFactory, StateReader, StateRootProvider, + PruneCheckpointReader, StageCheckpointReader, StateRootProvider, StorageSettingsCache, + TryIntoHistoricalStateProvider, }; use reth_storage_overlay::OverlayManager; use reth_tasks::utils::increase_thread_priority; @@ -819,16 +820,15 @@ fn published_sparse_trie_anchor_hash( impl StateRootStrategy for DefaultStateRootStrategy where N: NodePrimitives, - P: DatabaseProviderFactory - + BlockReader
- + StateProviderFactory - + StateReader - + Clone + P: DatabaseProviderFactory + Clone + 'static, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + 'static, OverlayStateProviderFactory: DatabaseProviderROFactory + Clone - + Send - + Sync + 'static, Evm: ConfigureEvm + 'static, { @@ -991,7 +991,13 @@ struct SynchronousStateRootJob { impl StateRootJob for SynchronousStateRootJob where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, + P: DatabaseProviderFactory + Clone + 'static, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, { fn name(&self) -> &'static str { "synchronous" @@ -1024,12 +1030,15 @@ struct SparseTrieStateRootJob { impl SparseTrieStateRootJob where N: NodePrimitives, - P: StateProviderFactory + Clone + Send + Sync + 'static, - P: BlockReader + StateReader, + P: DatabaseProviderFactory + Clone + 'static, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, OverlayStateProviderFactory: DatabaseProviderROFactory + Clone - + Send - + Sync + 'static, { fn serial_fallback( @@ -1131,11 +1140,15 @@ where impl StateRootJob for SparseTrieStateRootJob where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, + P: DatabaseProviderFactory + Clone + 'static, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, OverlayStateProviderFactory: DatabaseProviderROFactory + Clone - + Send - + Sync + 'static, { fn name(&self) -> &'static str { @@ -1232,7 +1245,13 @@ fn compare_trie_updates_with_serial( ) -> bool where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone, + P: DatabaseProviderFactory, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, OverlayStateProviderFactory: DatabaseProviderROFactory, { diff --git a/crates/engine/tree/src/tree/txpool_prewarm/mod.rs b/crates/engine/tree/src/tree/txpool_prewarm/mod.rs index 3070148fd3d..091058b0c74 100644 --- a/crates/engine/tree/src/tree/txpool_prewarm/mod.rs +++ b/crates/engine/tree/src/tree/txpool_prewarm/mod.rs @@ -9,7 +9,10 @@ use alloy_consensus::transaction::Recovered; use alloy_primitives::{Address, B256}; use reth_evm::{ConfigureEvm, EvmEnvFor}; use reth_primitives_traits::{NodePrimitives, TxTy}; -use reth_provider::{BlockReader, StateProviderFactory, StateReader}; +use reth_provider::{ + BlockNumReader, DatabaseProviderFactory, PruneCheckpointReader, StageCheckpointReader, + StorageSettingsCache, TryIntoHistoricalStateProvider, +}; use std::{fmt::Debug, sync::Arc}; /// Coordinates a long-lived worker and the latest completed immutable snapshot. @@ -34,7 +37,13 @@ where impl Handle where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone + Send + Sync + 'static, + P: DatabaseProviderFactory + 'static, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, Evm: ConfigureEvm + 'static, { /// Spawns the long-lived worker, which owns its mutable read cache and starts a fresh one for diff --git a/crates/engine/tree/src/tree/txpool_prewarm/worker.rs b/crates/engine/tree/src/tree/txpool_prewarm/worker.rs index 54ecd1ffb74..1400b68ae72 100644 --- a/crates/engine/tree/src/tree/txpool_prewarm/worker.rs +++ b/crates/engine/tree/src/tree/txpool_prewarm/worker.rs @@ -8,7 +8,10 @@ use alloy_primitives::B256; use crossbeam_channel::{Receiver, RecvTimeoutError, TryRecvError}; use reth_evm::ConfigureEvm; use reth_primitives_traits::NodePrimitives; -use reth_provider::{BlockReader, StateProviderFactory, StateReader}; +use reth_provider::{ + BlockNumReader, DatabaseProviderFactory, PruneCheckpointReader, StageCheckpointReader, + StorageSettingsCache, TryIntoHistoricalStateProvider, +}; use reth_revm::{cached::CachedReads, db::State}; use std::{ sync::Arc, @@ -63,7 +66,13 @@ where impl Worker where N: NodePrimitives, - P: BlockReader + StateProviderFactory + StateReader + Clone, + P: DatabaseProviderFactory, + P::Provider: BlockNumReader + + PruneCheckpointReader + + StageCheckpointReader + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, Evm: ConfigureEvm, { pub(super) fn new( @@ -315,6 +324,7 @@ mod tests { use reth_ethereum_primitives::{EthPrimitives, TransactionSigned}; use reth_evm_ethereum::EthEvmConfig; use reth_provider::test_utils::MockEthProvider; + use reth_stages_api::{StageCheckpoint, StageId}; use std::{ collections::{HashMap, VecDeque}, sync::atomic::{AtomicUsize, Ordering}, @@ -360,12 +370,16 @@ mod tests { /// Points the worker at `parent_hash`, as [`Handle::start`](super::super::Handle) does. fn start(&self, parent_hash: B256) { + let provider = MockEthProvider::default(); + provider.enable_database_provider(); + provider.add_header(parent_hash, Default::default()); + provider.add_stage_checkpoint(StageId::Finish, StageCheckpoint::new(0)); let job = Job { evm_env: Default::default(), provider_builder: StateProviderBuilder::new( - MockEthProvider::default(), + provider, parent_hash, - None, + reth_storage_overlay::OverlayManager::default(), ), }; self.commands.send(Command::Start { parent_hash, job }).unwrap(); diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 4782d6fded9..f001aca3279 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,3 +1,4 @@ +use super::state::latest::LatestStateProvider; use crate::{ providers::{ ConsistentProvider, OverlayStateProvider, OverlayStateProviderFactory, ProviderNodeTypes, @@ -32,9 +33,10 @@ use reth_static_file_types::StaticFileSegment; use reth_storage_api::{ BlockBodyIndicesProvider, NodePrimitivesProvider, RangeEnd, RangeResponse, RangeResult, StateRangeProvider, StateRangeProviderFactory, StateRangeView, StorageChangeSetReader, - StorageRangeResult, + StorageRangeResult, TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::ProviderResult; +use reth_storage_overlay::{anchor_for_parent, AnchorForParent}; use reth_trie::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, metrics::TrieRootMetrics, @@ -152,9 +154,20 @@ impl BlockchainProvider { &self, state: &BlockState, ) -> ProviderResult> { - let anchor_hash = state.anchor().hash; - let latest_historical = self.database.history_by_block_hash(anchor_hash)?; - Ok(state.state_provider(latest_historical)) + let provider = self.database.provider()?; + let anchor = + anchor_for_parent(state.hash(), state.chain().map(|state| state.block()), &provider)?; + + let (historical, overlay): (StateProviderBox, _) = match anchor { + AnchorForParent::NoReverts { overlay, .. } => { + (Box::new(LatestStateProvider::new(provider)), overlay) + } + AnchorForParent::RevertsRequired { anchor, overlay, .. } => { + (provider.try_into_history_at_block(anchor.number)?, overlay) + } + }; + + Ok(MemoryOverlayStateProvider::new(historical, overlay)) } /// Returns a cursor-backed state view for a state root still only in canonical in-memory diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 3fa34190c1a..0a56b94748d 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -38,6 +38,7 @@ use reth_storage_api::{ BlockBodyIndicesProvider, BytecodeReader, DBProvider, DatabaseProviderFactory, HashedPostStateProvider, NodePrimitivesProvider, StageCheckpointReader, StateProofProvider, StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, + TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::{ConsistentViewError, ProviderError, ProviderResult}; use reth_trie::{ @@ -73,8 +74,12 @@ pub struct MockEthProvider>>, /// Local block body indices store pub block_body_indices: Arc>>, + /// Local stage checkpoints + stage_checkpoints: Arc>>, /// Local BAL store handle pub bal_store: BalStoreHandle, + /// Whether database provider creation succeeds. + database_provider_available: Arc, /// Whether snap state reads should fail for handler error-path tests. snap_state_reads_fail: Arc, /// Whether a snap state range view is available. @@ -125,7 +130,9 @@ where chain_spec: self.chain_spec.clone(), state_roots: self.state_roots.clone(), block_body_indices: self.block_body_indices.clone(), + stage_checkpoints: self.stage_checkpoints.clone(), bal_store: self.bal_store.clone(), + database_provider_available: self.database_provider_available.clone(), snap_state_reads_fail: self.snap_state_reads_fail.clone(), snap_state_range_available: self.snap_state_range_available.clone(), snap_state_range_resolutions: self.snap_state_range_resolutions.clone(), @@ -152,7 +159,9 @@ impl MockEthProvider { chain_spec: Arc::new(reth_chainspec::ChainSpecBuilder::mainnet().build()), state_roots: Default::default(), block_body_indices: Default::default(), + stage_checkpoints: Default::default(), bal_store: Default::default(), + database_provider_available: Default::default(), snap_state_reads_fail: Default::default(), snap_state_range_available: Default::default(), snap_state_range_resolutions: Default::default(), @@ -169,6 +178,11 @@ impl MockEthProvider { } impl MockEthProvider { + /// Allows database provider creation to return this mock. + pub fn enable_database_provider(&self) { + self.database_provider_available.store(true, Ordering::Relaxed); + } + /// Makes snap state reads return provider errors when `fail` is true. pub fn set_snap_state_reads_fail(&self, fail: bool) { self.snap_state_reads_fail.store(fail, Ordering::Relaxed); @@ -296,6 +310,11 @@ impl MockEthProvider { self.block_body_indices.lock().insert(block_number, indices); } + /// Adds a stage checkpoint to the local store. + pub fn add_stage_checkpoint(&self, id: StageId, checkpoint: StageCheckpoint) { + self.stage_checkpoints.lock().insert(id, checkpoint); + } + /// Add state root to local state root store pub fn add_state_root(&self, state_root: B256) { self.state_roots.lock().push(state_root); @@ -311,7 +330,9 @@ impl MockEthProvider { chain_spec: Arc::new(chain_spec), state_roots: self.state_roots, block_body_indices: self.block_body_indices, + stage_checkpoints: self.stage_checkpoints, bal_store: self.bal_store, + database_provider_available: self.database_provider_available, snap_state_reads_fail: self.snap_state_reads_fail, snap_state_range_available: self.snap_state_range_available, snap_state_range_resolutions: self.snap_state_range_resolutions, @@ -475,11 +496,19 @@ impl DatabaseProvi type ProviderRW = Self; fn database_provider_ro(&self) -> ProviderResult { - Err(ConsistentViewError::Syncing { best_block: GotExpected::new(0, 0) }.into()) + if self.database_provider_available.load(Ordering::Relaxed) { + Ok(self.clone()) + } else { + Err(ConsistentViewError::Syncing { best_block: GotExpected::new(0, 0) }.into()) + } } fn database_provider_rw(&self) -> ProviderResult { - Err(ConsistentViewError::Syncing { best_block: GotExpected::new(0, 0) }.into()) + if self.database_provider_available.load(Ordering::Relaxed) { + Ok(self.clone()) + } else { + Err(ConsistentViewError::Syncing { best_block: GotExpected::new(0, 0) }.into()) + } } } @@ -963,8 +992,8 @@ impl AccountReader for MockEthProvide impl StageCheckpointReader for MockEthProvider { - fn get_stage_checkpoint(&self, _id: StageId) -> ProviderResult> { - Ok(None) + fn get_stage_checkpoint(&self, id: StageId) -> ProviderResult> { + Ok(self.stage_checkpoints.lock().get(&id).copied()) } fn get_stage_checkpoint_progress(&self, _id: StageId) -> ProviderResult>> { @@ -972,7 +1001,12 @@ impl StageCheckpointReader } fn get_all_checkpoints(&self) -> ProviderResult> { - Ok(vec![]) + Ok(self + .stage_checkpoints + .lock() + .iter() + .map(|(id, checkpoint)| (id.to_string(), *checkpoint)) + .collect()) } } @@ -1200,6 +1234,17 @@ impl StatePr } } +impl + TryIntoHistoricalStateProvider for MockEthProvider +{ + fn try_into_history_at_block( + self, + block_number: BlockNumber, + ) -> ProviderResult { + self.history_by_block_number(block_number) + } +} + impl BlockBodyIndicesProvider for MockEthProvider { diff --git a/crates/storage/provider/src/traits/full.rs b/crates/storage/provider/src/traits/full.rs index fb0a18565a2..1f4bdcd7760 100644 --- a/crates/storage/provider/src/traits/full.rs +++ b/crates/storage/provider/src/traits/full.rs @@ -9,7 +9,10 @@ use reth_chain_state::{ CanonStateSubscriptions, ForkChoiceSubscriptions, PersistedBlockSubscriptions, }; use reth_node_types::{BlockTy, HeaderTy, NodeTypesWithDB, ReceiptTy, TxTy}; -use reth_storage_api::{NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache}; +use reth_storage_api::{ + NodePrimitivesProvider, StorageChangeSetReader, StorageSettingsCache, + TryIntoHistoricalStateProvider, +}; use std::fmt::Debug; /// Helper trait to unify all provider traits for simplicity. @@ -21,7 +24,9 @@ pub trait FullProvider: + PruneCheckpointReader + ChangeSetReader + StorageChangeSetReader - + StorageSettingsCache, + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, > + NodePrimitivesProvider + StaticFileProviderFactory + RocksDBProviderFactory @@ -57,7 +62,9 @@ impl FullProvider for T where + PruneCheckpointReader + ChangeSetReader + StorageChangeSetReader - + StorageSettingsCache, + + StorageSettingsCache + + TryIntoHistoricalStateProvider + + 'static, > + NodePrimitivesProvider + StaticFileProviderFactory + RocksDBProviderFactory diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index b5f356a6f89..28aeb5f857e 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -9,7 +9,7 @@ use crate::{ PruneCheckpointReader, ReceiptProvider, ReceiptProviderIdExt, StageCheckpointReader, StateProofProvider, StateProvider, StateProviderBox, StateProviderFactory, StateRangeProviderFactory, StateRangeView, StateReader, StateRootProvider, StorageRootProvider, - TransactionVariant, TransactionsProvider, + TransactionVariant, TransactionsProvider, TryIntoHistoricalStateProvider, }; #[cfg(feature = "db-api")] @@ -627,6 +627,17 @@ impl StateProviderFactory for NoopP } } +impl TryIntoHistoricalStateProvider + for NoopProvider +{ + fn try_into_history_at_block( + self, + block_number: BlockNumber, + ) -> ProviderResult { + self.history_by_block_number(block_number) + } +} + impl StageCheckpointReader for NoopProvider { fn get_stage_checkpoint(&self, _id: StageId) -> ProviderResult> { Ok(None) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index b66f43d7840..7096e1681c5 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -1,11 +1,12 @@ use crate::OverlayManager; use alloy_eips::BlockNumHash; -use alloy_primitives::{BlockHash, BlockNumber, B256}; +use alloy_primitives::{BlockHash, B256}; use metrics::{Counter, Histogram}; +use reth_chain_state::ExecutedBlock; use reth_errors::{ProviderError, ProviderResult}; use reth_ethereum_primitives::EthPrimitives; use reth_metrics::Metrics; -use reth_primitives_traits::NodePrimitives; +use reth_primitives_traits::{AlloyBlockHeader, NodePrimitives}; use reth_prune_types::PruneSegment; use reth_stages_types::StageId; use reth_storage_api::{ @@ -14,7 +15,6 @@ use reth_storage_api::{ }; use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; use std::{ - ops::RangeInclusive, sync::Arc, time::{Duration, Instant}, }; @@ -110,6 +110,46 @@ impl OverlayBuilder { self } + /// Returns the durable anchor to use for this builder's parent. + pub fn anchor_at_parent( + &self, + provider: &Provider, + ) -> ProviderResult> + where + Provider: StageCheckpointReader + BlockNumReader + PruneCheckpointReader, + { + let (partial_state_trie, finish) = database_state_frontiers(provider)?; + self.anchor_at_parent_with_frontiers(provider, partial_state_trie, finish) + } + + /// Returns the durable anchor to use for this builder's parent using known frontiers. + pub fn anchor_at_parent_with_frontiers( + &self, + provider: &Provider, + partial_state_trie: BlockNumHash, + finish: BlockNumHash, + ) -> ProviderResult> + where + Provider: BlockNumReader + PruneCheckpointReader, + { + match &self.overlay_source { + Some(OverlaySource::Managed) => anchor_for_parent_with_frontiers( + self.parent_hash, + self.overlay_manager.parent_chain(self.parent_hash), + partial_state_trie, + finish, + provider, + ), + _ => anchor_for_parent_with_frontiers( + self.parent_hash, + std::iter::empty(), + partial_state_trie, + finish, + provider, + ), + } + } + /// Builds the effective overlay for the given provider. #[instrument(level = "debug", target = "storage::overlay", skip_all)] pub fn build_overlay(&self, provider: &Provider) -> ProviderResult @@ -146,7 +186,6 @@ impl OverlayBuilder { + StorageChangeSetReader + DBProvider + BlockNumReader - + StageCheckpointReader + PruneCheckpointReader + StorageSettingsCache, { @@ -154,126 +193,118 @@ impl OverlayBuilder { let retrieve_hashed_state_reverts_duration; let trie_updates_total_len; let hashed_state_updates_total_len; - let anchor_hash = match &self.overlay_source { - Some(OverlaySource::Managed) => { - let parent_is_persisted = provider - .convert_hash_or_number(self.parent_hash.into())? - .is_some_and(|parent_number| parent_number <= state_trie_tip_block.number); - if parent_is_persisted { - self.parent_hash - } else { - self.overlay_manager - .anchor_for_parent(self.parent_hash, state_trie_tip_block.hash) - .ok_or(ProviderError::BlockHashNotFound(self.parent_hash))? - } - } - _ => self.parent_hash, - }; + + let anchor_for_parent = + self.anchor_at_parent_with_frontiers(provider, state_trie_tip_block, finish_tip_block)?; // Collect any reverts which are required to bring the DB view back to the anchor hash. - let (trie_updates, hashed_post_state) = if let Some(revert_blocks) = - self.reverts_required(provider, state_trie_tip_block, finish_tip_block, anchor_hash)? - { - debug!( - target: "storage::overlay", - ?revert_blocks, - %anchor_hash, - "Collecting trie reverts for overlay state provider" - ); + let (trie_updates, hashed_post_state) = match anchor_for_parent { + AnchorForParent::RevertsRequired { anchor, finish, .. } => { + let revert_blocks = anchor.number + 1..=finish.number; + + debug!( + target: "storage::overlay", + ?revert_blocks, + ?anchor, + "Collecting trie reverts for overlay state provider" + ); - let trie_reverts = { - let _guard = - debug_span!(target: "storage::overlay", "retrieving_trie_reverts").entered(); - let start = Instant::now(); - let accumulated_reverts = self - .overlay_manager - .get_or_compute_cached_changesets_range(provider, revert_blocks.clone())?; - retrieve_trie_reverts_duration = start.elapsed(); - accumulated_reverts - }; - - let mut hashed_state_reverts = { - let _guard = - debug_span!(target: "storage::overlay", "retrieving_hashed_state_reverts") + let trie_reverts = { + let _guard = debug_span!(target: "storage::overlay", "retrieving_trie_reverts") .entered(); - let start = Instant::now(); - let res = reth_trie_db::from_reverts_auto(provider, revert_blocks)?; - retrieve_hashed_state_reverts_duration = start.elapsed(); - res - }; - - // Resolve overlays and extend reverts with them. If reverts are empty, use overlays - // directly to avoid cloning. - let (overlay_trie, overlay_state) = self.resolve_overlays(anchor_hash)?; - - let trie_updates = if trie_reverts.is_empty() { - overlay_trie - } else if !overlay_trie.is_empty() { - let mut trie_reverts = (*trie_reverts).clone(); - trie_reverts.extend_ref_and_sort(&overlay_trie); - Arc::new(trie_reverts) - } else { - trie_reverts - }; - - let hashed_state_updates = if hashed_state_reverts.is_empty() { - overlay_state - } else if !overlay_state.is_empty() { - hashed_state_reverts.extend_ref_and_sort(&overlay_state); - Arc::new(hashed_state_reverts) - } else { - Arc::new(hashed_state_reverts) - }; - - trie_updates_total_len = trie_updates.total_len(); - hashed_state_updates_total_len = hashed_state_updates.total_len(); - - debug!( - target: "storage::overlay", - num_trie_updates = ?trie_updates_total_len, - num_state_updates = ?hashed_state_updates_total_len, - %anchor_hash, - "Reverted to anchor block", - ); + let start = Instant::now(); + let accumulated_reverts = self + .overlay_manager + .get_or_compute_cached_changesets_range(provider, revert_blocks.clone())?; + retrieve_trie_reverts_duration = start.elapsed(); + accumulated_reverts + }; + + let mut hashed_state_reverts = { + let _guard = + debug_span!(target: "storage::overlay", "retrieving_hashed_state_reverts") + .entered(); + let start = Instant::now(); + let res = reth_trie_db::from_reverts_auto(provider, revert_blocks)?; + retrieve_hashed_state_reverts_duration = start.elapsed(); + res + }; + + // Resolve overlays and extend reverts with them. If reverts are empty, use overlays + // directly to avoid cloning. + let (overlay_trie, overlay_state) = self.resolve_overlays(anchor.hash)?; + + let trie_updates = if trie_reverts.is_empty() { + overlay_trie + } else if !overlay_trie.is_empty() { + let mut trie_reverts = (*trie_reverts).clone(); + trie_reverts.extend_ref_and_sort(&overlay_trie); + Arc::new(trie_reverts) + } else { + trie_reverts + }; + + let hashed_state_updates = if hashed_state_reverts.is_empty() { + overlay_state + } else if !overlay_state.is_empty() { + hashed_state_reverts.extend_ref_and_sort(&overlay_state); + Arc::new(hashed_state_reverts) + } else { + Arc::new(hashed_state_reverts) + }; + + trie_updates_total_len = trie_updates.total_len(); + hashed_state_updates_total_len = hashed_state_updates.total_len(); - (trie_updates, hashed_state_updates) - } else { - // If no reverts are needed, use the manager overlay directly unless the reused sparse - // trie already covers both durable frontiers through the requested parent. - if self.should_skip_overlay_for_reused_sparse_trie( - state_trie_tip_block.hash, - finish_tip_block.hash, - ) { debug!( target: "storage::overlay", - parent_hash = %self.parent_hash, - state_trie_tip_hash = %state_trie_tip_block.hash, - finish_tip_hash = %finish_tip_block.hash, - sparse_trie_anchor_hash = ?self.reused_sparse_trie_anchor_hash, - "Skipping overlay construction because reused sparse trie covers durable frontiers to parent" + num_trie_updates = ?trie_updates_total_len, + num_state_updates = ?hashed_state_updates_total_len, + ?anchor, + "Reverted to anchor block", ); - self.metrics.sparse_trie_overlay_skips.increment(1); - - return Ok(Overlay::empty()) + (trie_updates, hashed_state_updates) } + AnchorForParent::NoReverts { anchor, .. } => { + // If no reverts are needed, use the manager overlay directly unless the reused + // sparse trie already covers both durable frontiers through the + // requested parent. + if self.should_skip_overlay_for_reused_sparse_trie( + state_trie_tip_block.hash, + finish_tip_block.hash, + ) { + debug!( + target: "storage::overlay", + parent_hash = %self.parent_hash, + state_trie_tip_hash = %state_trie_tip_block.hash, + finish_tip_hash = %finish_tip_block.hash, + sparse_trie_anchor_hash = ?self.reused_sparse_trie_anchor_hash, + "Skipping overlay construction because reused sparse trie covers durable frontiers to parent" + ); + + self.metrics.sparse_trie_overlay_skips.increment(1); + + return Ok(Overlay::empty()) + } - let (trie_updates, hashed_post_state) = self.resolve_overlays(anchor_hash)?; + let (trie_updates, hashed_post_state) = self.resolve_overlays(anchor.hash)?; - retrieve_trie_reverts_duration = Duration::ZERO; - retrieve_hashed_state_reverts_duration = Duration::ZERO; - trie_updates_total_len = trie_updates.total_len(); - hashed_state_updates_total_len = hashed_post_state.total_len(); + retrieve_trie_reverts_duration = Duration::ZERO; + retrieve_hashed_state_reverts_duration = Duration::ZERO; + trie_updates_total_len = trie_updates.total_len(); + hashed_state_updates_total_len = hashed_post_state.total_len(); - debug!( - target: "storage::overlay", - num_trie_updates = trie_updates_total_len, - num_state_updates = hashed_state_updates_total_len, - %anchor_hash, - "Built overlay directly from durable frontier" - ); + debug!( + target: "storage::overlay", + num_trie_updates = trie_updates_total_len, + num_state_updates = hashed_state_updates_total_len, + ?anchor, + "Built overlay directly from durable frontier" + ); - (trie_updates, hashed_post_state) + (trie_updates, hashed_post_state) + } }; self.metrics @@ -345,93 +376,6 @@ impl OverlayBuilder { _ => false, } } - - /// Returns whether or not it is required to collect reverts, and validates that there are - /// sufficient changesets to revert to the requested block number if so. - /// - /// Takes into account both the stage checkpoint and the prune checkpoint to determine the - /// available data range. - fn reverts_required( - &self, - provider: &Provider, - state_trie_tip_block: BlockNumHash, - finish_tip_block: BlockNumHash, - anchor_hash: BlockHash, - ) -> ProviderResult>> - where - Provider: BlockNumReader + PruneCheckpointReader, - { - let anchor_number = provider - .convert_hash_or_number(anchor_hash.into())? - .ok_or(ProviderError::BlockHashNotFound(anchor_hash))?; - let canonical_anchor_hash = provider - .convert_number(anchor_number.into())? - .ok_or_else(|| ProviderError::HeaderNotFound(anchor_number.into()))?; - if canonical_anchor_hash != anchor_hash { - return Err(ProviderError::other(std::io::Error::other(format!( - "overlay anchor {anchor_hash} is not on the durable finish chain at block {anchor_number} (found {canonical_anchor_hash})", - )))) - } - - // With no partial-persistence gap, a parent at the Finish tip is already exposed by the - // database without an overlay or reverts. - if state_trie_tip_block.hash == finish_tip_block.hash && - finish_tip_block.hash == anchor_hash - { - return Ok(None) - } - - // The database is a hybrid view while the state/trie and Finish frontiers differ. A - // manager overlay can use that view directly only when its anchor-to-parent path covers - // both frontiers; otherwise the database must first be reverted to the overlay anchor. - if matches!(&self.overlay_source, Some(OverlaySource::Managed)) && - self.overlay_manager.contains_hash( - self.parent_hash, - anchor_hash, - state_trie_tip_block.hash, - ) && - self.overlay_manager.contains_hash( - self.parent_hash, - anchor_hash, - finish_tip_block.hash, - ) - { - return Ok(None) - } - - if anchor_number > state_trie_tip_block.number { - return Err(ProviderError::other(std::io::Error::other(format!( - "overlay anchor #{} ({}) is after partial state trie frontier #{} ({}); missing trie updates for blocks #{}..=#{}", - anchor_number, - anchor_hash, - state_trie_tip_block.number, - state_trie_tip_block.hash, - state_trie_tip_block.number + 1, - anchor_number, - )))) - } - - // Check history prune checkpoints to determine the earliest anchor that can be - // reconstructed. A checkpoint at block N means changesets starting at N + 1 are available, - // which is sufficient to reconstruct the state at N. Both account and storage changesets - // are required, so the later checkpoint determines the lower bound. - let account_history = provider - .get_prune_checkpoint(PruneSegment::AccountHistory)? - .and_then(|checkpoint| checkpoint.block_number); - let storage_history = provider - .get_prune_checkpoint(PruneSegment::StorageHistory)? - .and_then(|checkpoint| checkpoint.block_number); - let lower_bound = account_history.max(storage_history).unwrap_or_default(); - let available_range = lower_bound..=finish_tip_block.number; - if !available_range.contains(&anchor_number) { - return Err(ProviderError::InsufficientChangesets { - requested: anchor_number, - available: available_range, - }) - } - - Ok(Some(anchor_number + 1..=finish_tip_block.number)) - } } /// Returns the highest blocks whose state/trie data and non-state/trie data are durably @@ -480,6 +424,165 @@ struct OverlayBuilderMetrics { sparse_trie_overlay_skips: Counter, } +fn anchor_for_parent_in( + parent_hash: B256, + mut in_mem_chain: impl Iterator>, + preferred_anchor: B256, +) -> B256 { + if parent_hash == preferred_anchor { + return parent_hash + } + + let mut hash = parent_hash; + + loop { + let Some(block) = in_mem_chain.next() else { return hash }; + let block_parent_hash = block.recovered_block().parent_hash(); + + if block_parent_hash == preferred_anchor { + return block_parent_hash + } + hash = block_parent_hash; + } +} + +/// Describes whether an overlay must revert the database before using its anchor. +#[derive(Debug)] +pub enum AnchorForParent { + /// The in-memory chain covers the durable frontiers through this anchor. + NoReverts { + /// Block to anchor the overlay to. + anchor: BlockNumHash, + /// In-memory blocks from `parent_hash` through, but excluding, `anchor`. + overlay: Vec>, + }, + /// The database must be reverted from `finish` to `anchor` first. + RevertsRequired { + /// Block to anchor the overlay to. + anchor: BlockNumHash, + /// Current Finish frontier. + finish: BlockNumHash, + /// In-memory blocks from `parent_hash` through, but excluding, `anchor`. + overlay: Vec>, + }, +} + +/// Returns the anchor block to use for the target parent and a chain of in-memory blocks. +/// +/// # Arguments +/// * `parent`: The block whose post-state is being targeted. +/// * `in_mem_chain`: Yields the in-memory blocks in the chain, starting at `parent_hash`. +/// * `provider`: Used to resolve the durable frontiers and check changeset availability. +pub fn anchor_for_parent( + parent_hash: B256, + in_mem_chain: impl Iterator>, + provider: &Provider, +) -> ProviderResult> +where + N: NodePrimitives, + Provider: StageCheckpointReader + BlockNumReader + PruneCheckpointReader, +{ + let (partial_state_trie, finish) = database_state_frontiers(provider)?; + anchor_for_parent_with_frontiers( + parent_hash, + in_mem_chain, + partial_state_trie, + finish, + provider, + ) +} + +/// Returns the anchor block to use for the target parent and a chain of in-memory blocks using +/// known durable frontiers. +/// +/// # Arguments +/// * `parent`: The block whose post-state is being targeted. +/// * `in_mem_chain`: Yields the in-memory blocks in the chain, starting at `parent_hash`. +/// * `partial_state_trie`: The durable state/trie frontier. +/// * `finish`: The durable Finish frontier. +/// * `provider`: Used to resolve the parent and check changeset availability. +pub fn anchor_for_parent_with_frontiers( + parent_hash: B256, + in_mem_chain: impl Iterator>, + partial_state_trie: BlockNumHash, + finish: BlockNumHash, + provider: &Provider, +) -> ProviderResult> +where + N: NodePrimitives, + Provider: BlockNumReader + PruneCheckpointReader, +{ + use std::io::Error; + + let persisted_parent = provider + .block_number(parent_hash)? + .filter(|&parent_number| parent_number <= partial_state_trie.number); + + let mut finish_seen = parent_hash == finish.hash; + let (anchor, overlay) = if let Some(parent_number) = persisted_parent { + (BlockNumHash::new(parent_number, parent_hash), Vec::new()) + } else { + let mut overlay = Vec::new(); + let mut in_mem_chain = in_mem_chain.inspect(|block| { + finish_seen |= block.recovered_block().hash() == finish.hash; + overlay.push(block.clone()); + }); + + let anchor_hash = + anchor_for_parent_in(parent_hash, &mut in_mem_chain, partial_state_trie.hash); + let anchor = if anchor_hash == partial_state_trie.hash { + BlockNumHash::new(partial_state_trie.number, anchor_hash) + } else { + let anchor_number = provider + .convert_hash_or_number(anchor_hash.into())? + .ok_or(ProviderError::BlockHashNotFound(anchor_hash))?; + BlockNumHash::new(anchor_number, anchor_hash) + }; + (anchor, overlay) + }; + + finish_seen |= anchor.hash == finish.hash; + + if anchor.number > partial_state_trie.number { + return Err(ProviderError::other(Error::other(format!( + "overlay anchor #{} ({}) is after partial state trie frontier #{} ({}); missing trie updates for blocks #{}..=#{}", + anchor.number, + anchor.hash, + partial_state_trie.number, + partial_state_trie.hash, + partial_state_trie.number + 1, + anchor.number, + )))) + } + + // If the Finish block (db tip) was seen in the in-memory chain then we know that anchor is on + // the same chain as partial_state_trie as well. Given that anchor <= partial_state_trie, we can + // be sure that the in-memory chain is a superset of partial_state_trie+1..finish, and therefore + // can be used without reverts. + if finish_seen { + return Ok(AnchorForParent::NoReverts { anchor, overlay }) + } + + // Otherwise reverts are required; we check the changesets to make sure they are actually + // available before signaling that they are required. + let account_history = provider + .get_prune_checkpoint(PruneSegment::AccountHistory)? + .and_then(|checkpoint| checkpoint.block_number); + let storage_history = provider + .get_prune_checkpoint(PruneSegment::StorageHistory)? + .and_then(|checkpoint| checkpoint.block_number); + let lower_bound = account_history.max(storage_history).unwrap_or_default(); + let available_range = lower_bound..=finish.number; + if !available_range.contains(&anchor.number) { + return Err(ProviderError::InsufficientChangesets { + requested: anchor.number, + available: available_range, + }) + } + + Ok(AnchorForParent::RevertsRequired { anchor, finish, overlay }) +} + #[cfg(test)] mod tests { use super::*; @@ -492,11 +595,10 @@ mod tests { BlockWriter, ProviderFactory, }; #[cfg(feature = "partial-persistence")] - use reth_prune_types::{PruneCheckpoint, PruneMode}; #[cfg(feature = "partial-persistence")] use reth_stages_types::{FinishCheckpoint, StageCheckpoint}; #[cfg(feature = "partial-persistence")] - use reth_storage_api::{PruneCheckpointWriter, StageCheckpointWriter}; + use reth_storage_api::StageCheckpointWriter; use reth_trie::{BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles}; fn with_unique_trie_data( @@ -603,14 +705,15 @@ mod tests { #[cfg(feature = "partial-persistence")] #[test] - fn managed_overlay_uses_persisted_parent_even_if_retained() { - let (factory, blocks) = setup_frontiers(2, 3); + fn managed_overlay_skips_when_finish_is_the_anchor() { + let (factory, blocks) = setup_frontiers(3, 3); let manager = OverlayManager::default(); - manager.insert_block(blocks[1].clone()); + manager.insert_block(blocks[4].clone()); let provider = factory.provider().unwrap(); let overlay = manager - .overlay_builder(blocks[1].recovered_block().hash()) + .overlay_builder(blocks[4].recovered_block().hash()) + .with_skip_overlay_for_reused_sparse_trie(blocks[3].recovered_block().hash()) .build_overlay(&provider) .unwrap(); @@ -620,83 +723,27 @@ mod tests { #[cfg(feature = "partial-persistence")] #[test] - fn parent_inside_finish_gap_reverts_to_state_trie_frontier() { - let (factory, blocks) = setup_frontiers(1, 3); - let manager = OverlayManager::default(); - manager.insert_block(blocks[2].clone()); - let provider = factory.provider().unwrap(); - let builder = manager.overlay_builder(blocks[2].recovered_block().hash()); - let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); - let anchor_hash = blocks[1].recovered_block().hash(); - let revert_blocks = - builder.reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash).unwrap(); - - assert_eq!(revert_blocks, Some(2..=3)); - } - - #[cfg(feature = "partial-persistence")] - #[test] - fn anchor_at_prune_checkpoint_has_sufficient_changesets() { - let (factory, blocks) = setup_frontiers(1, 3); - let provider_rw = factory.provider_rw().unwrap(); - provider_rw - .save_prune_checkpoint( - PruneSegment::AccountHistory, - PruneCheckpoint { - block_number: Some(blocks[1].block_number()), - tx_number: None, - prune_mode: PruneMode::Full, - }, - ) - .unwrap(); - provider_rw.commit().unwrap(); - - let manager = OverlayManager::default(); - manager.insert_block(blocks[2].clone()); - let provider = factory.provider().unwrap(); - let builder = manager.overlay_builder(blocks[2].recovered_block().hash()); - let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); - let anchor_hash = blocks[1].recovered_block().hash(); - let revert_blocks = - builder.reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash).unwrap(); - - assert_eq!(revert_blocks, Some(2..=3)); - } - - #[cfg(feature = "partial-persistence")] - #[test] - fn storage_history_checkpoint_limits_available_anchor() { - let (factory, blocks) = setup_frontiers(1, 3); - let provider_rw = factory.provider_rw().unwrap(); - provider_rw - .save_prune_checkpoint( - PruneSegment::StorageHistory, - PruneCheckpoint { - block_number: Some(blocks[2].block_number()), - tx_number: None, - prune_mode: PruneMode::Full, - }, - ) - .unwrap(); - provider_rw.commit().unwrap(); - + fn managed_overlay_uses_persisted_parent_even_if_retained() { + let (factory, blocks) = setup_frontiers(2, 3); let manager = OverlayManager::default(); - manager.insert_block(blocks[2].clone()); + manager.insert_block(blocks[1].clone()); let provider = factory.provider().unwrap(); - let builder = manager.overlay_builder(blocks[2].recovered_block().hash()); - let (state_trie_tip, finish_tip) = database_state_frontiers(&provider).unwrap(); - let anchor_hash = blocks[1].recovered_block().hash(); - let error = builder - .reverts_required(&provider, state_trie_tip, finish_tip, anchor_hash) - .unwrap_err(); - - match error { - ProviderError::InsufficientChangesets { requested, available } => { - assert_eq!(requested, blocks[1].block_number()); - assert_eq!(available, blocks[2].block_number()..=blocks[3].block_number()); + let builder = manager.overlay_builder(blocks[1].recovered_block().hash()); + match builder.anchor_at_parent(&provider).unwrap() { + AnchorForParent::RevertsRequired { anchor, finish, overlay } => { + assert_eq!(anchor, blocks[1].recovered_block().num_hash()); + assert_eq!(finish, blocks[3].recovered_block().num_hash()); + assert!(overlay.is_empty()); + } + AnchorForParent::NoReverts { .. } => { + panic!("persisted parent below Finish must require reverts") } - error => panic!("unexpected error: {error}"), } + + let overlay = builder.build_overlay(&provider).unwrap(); + + assert!(overlay.hashed_post_state.is_empty()); + assert!(overlay.trie_updates.is_empty()); } #[cfg(feature = "partial-persistence")] @@ -727,7 +774,7 @@ mod tests { .build_overlay(&provider) .unwrap_err(); - assert!(matches!(error, ProviderError::BlockHashNotFound(hash) if hash == parent_hash)); + assert!(error.to_string().contains("is after partial state trie frontier")); } #[test] diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index b53e79b748d..7f677b3df8b 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -240,11 +240,8 @@ impl OverlayManager { if removed_blocks > 0 { let overlays_before = self.overlays.len(); - let blocks = Arc::clone(&self.blocks); self.overlays.retain(|key, _| { - key.tip_hash != key.anchor_hash && - Self::anchor_for_parent_in(blocks.as_ref(), key.tip_hash, key.anchor_hash) == - Some(key.anchor_hash) + self.contains_hash(key.tip_hash, key.anchor_hash, key.anchor_hash) }); pruned_overlays = overlays_before.saturating_sub(self.overlays.len()); span.record("pruned_overlays", pruned_overlays); @@ -265,7 +262,7 @@ impl OverlayManager { skip_all, fields(tip_hash = %parent_hash, anchor_hash = %anchor_hash) )] - pub fn overlay_for_parent( + pub(crate) fn overlay_for_parent( &self, parent_hash: B256, anchor_hash: B256, @@ -394,12 +391,17 @@ impl OverlayManager { span.record("cache_reused", true); } - /// Returns `preferred_anchor` if it is on the parent chain, otherwise the first missing parent. - /// - /// Returns `None` if `parent_hash` is not `preferred_anchor` and the manager does not contain a - /// block for `parent_hash`, meaning there is no in-memory parent chain to inspect. - pub fn anchor_for_parent(&self, parent_hash: B256, preferred_anchor: B256) -> Option { - Self::anchor_for_parent_in(self.blocks.as_ref(), parent_hash, preferred_anchor) + /// Returns every in-memory block in the chain whose tip is `parent_hash`. + pub(crate) fn parent_chain( + &self, + parent_hash: B256, + ) -> impl Iterator> + '_ { + let mut hash = parent_hash; + std::iter::from_fn(move || { + let block = self.blocks.get(&hash)?; + hash = block.recovered_block().parent_hash(); + Some(block.clone()) + }) } /// Returns true if `hash` is in the parent chain segment from `anchor_hash` inclusive to @@ -420,29 +422,6 @@ impl OverlayManager { } } - fn anchor_for_parent_in( - blocks: &DashMap>, - parent_hash: B256, - preferred_anchor: B256, - ) -> Option { - if parent_hash == preferred_anchor { - return Some(preferred_anchor) - } - - let mut hash = parent_hash; - - loop { - let block_parent_hash = blocks.get(&hash)?.recovered_block().parent_hash(); - if block_parent_hash == preferred_anchor { - return Some(block_parent_hash) - } - if !blocks.contains_key(&block_parent_hash) { - return Some(block_parent_hash) - } - hash = block_parent_hash; - } - } - fn compute_overlay( &self, compute_input: ComputeOverlayInput, @@ -732,44 +711,6 @@ mod tests { assert!(Arc::ptr_eq(&short, &cached_short)); } - #[test] - fn returns_anchor_for_in_memory_parent() { - let manager = OverlayManager::default(); - let blocks = test_blocks(); - for block in &blocks { - manager.insert_block(block.clone()); - } - - assert_eq!( - manager.anchor_for_parent(blocks[2].recovered_block().hash(), B256::random()), - Some(blocks[0].recovered_block().parent_hash()) - ); - - manager.remove_blocks([blocks[0].recovered_block().hash()]); - assert_eq!( - manager.anchor_for_parent( - blocks[2].recovered_block().hash(), - blocks[0].recovered_block().hash() - ), - Some(blocks[0].recovered_block().hash()) - ); - } - - #[test] - fn prefers_anchor_in_parent_chain() { - let manager = OverlayManager::default(); - let blocks = test_blocks(); - for block in &blocks { - manager.insert_block(block.clone()); - } - - let db_tip_hash = blocks[1].recovered_block().hash(); - assert_eq!( - manager.anchor_for_parent(blocks[2].recovered_block().hash(), db_tip_hash), - Some(db_tip_hash) - ); - } - #[test] fn contains_hash_detects_hashes_from_anchor_to_parent() { let manager = OverlayManager::default(); From 8e34c4b545b24f5170fca2fee32c3b09601514fb Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Thu, 6 Aug 2026 17:34:28 +0530 Subject: [PATCH 105/148] chore(hive): add execute-blob tests in the hive workflow (#26606) --- .github/scripts/hive/build_simulators.sh | 7 +++++++ .github/scripts/hive/load_images.sh | 1 + .github/workflows/hive.yml | 10 ++++++++++ 3 files changed, 18 insertions(+) diff --git a/.github/scripts/hive/build_simulators.sh b/.github/scripts/hive/build_simulators.sh index 8f3b320403d..67de5435dd8 100755 --- a/.github/scripts/hive/build_simulators.sh +++ b/.github/scripts/hive/build_simulators.sh @@ -7,10 +7,12 @@ case "${fixture_variant}" in amsterdam) eels_fixtures="https://github.com/ethereum/execution-specs/releases/download/tests-glamsterdam-devnet@v7.2.1/fixtures_glamsterdam-devnet.tar.gz" eels_branch="devnets/glamsterdam/7" + eels_fork="Amsterdam" ;; osaka) eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz" eels_branch="mainnet" + eels_fork="Osaka" ;; *) echo "unknown hive fixture variant: ${fixture_variant}" @@ -36,6 +38,10 @@ echo "Building images" --sim.buildarg fixtures="${eels_fixtures}" \ --sim.buildarg branch="${eels_branch}" \ --sim.timelimit 1s || true & +./hive -client reth --sim "ethereum/eels/execute-blobs" \ + --sim.buildarg branch="${eels_branch}" \ + --sim.buildarg fork="${eels_fork}" \ + --sim.timelimit 1s || true & ./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true & ./hive -client reth --sim "devp2p" -sim.timelimit 1s || true & ./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true & @@ -53,6 +59,7 @@ docker save hive/simulators/ethereum/engine:latest -o ../hive_assets/engine.tar docker save hive/simulators/ethereum/rpc-compat:latest -o ../hive_assets/rpc_compat.tar & saving_pids+=( $! ) docker save hive/simulators/ethereum/eels/consume-engine:latest -o ../hive_assets/eels_engine.tar & saving_pids+=( $! ) docker save hive/simulators/ethereum/eels/consume-rlp:latest -o ../hive_assets/eels_rlp.tar & saving_pids+=( $! ) +docker save hive/simulators/ethereum/eels/execute-blobs:latest -o ../hive_assets/eels_blobs.tar & saving_pids+=( $! ) docker save hive/simulators/smoke/genesis:latest -o ../hive_assets/smoke_genesis.tar & saving_pids+=( $! ) docker save hive/simulators/smoke/network:latest -o ../hive_assets/smoke_network.tar & saving_pids+=( $! ) docker save hive/simulators/ethereum/sync:latest -o ../hive_assets/ethereum_sync.tar & saving_pids+=( $! ) diff --git a/.github/scripts/hive/load_images.sh b/.github/scripts/hive/load_images.sh index e7dd7c99f4a..f0c164bc949 100755 --- a/.github/scripts/hive/load_images.sh +++ b/.github/scripts/hive/load_images.sh @@ -13,6 +13,7 @@ IMAGES=( "/tmp/ethereum_sync.tar" "/tmp/eels_engine.tar" "/tmp/eels_rlp.tar" + "/tmp/eels_blobs.tar" "/tmp/reth_image.tar" ) diff --git a/.github/workflows/hive.yml b/.github/workflows/hive.yml index bd5e1d7c7db..860c54d71d1 100644 --- a/.github/workflows/hive.yml +++ b/.github/workflows/hive.yml @@ -196,6 +196,12 @@ jobs: limit: .*tests/frontier.* - sim: ethereum/eels/consume-rlp limit: .*tests/paris.* + + # execute-blobs + - sim: ethereum/eels/execute-blobs + limit: .*tests/amsterdam.* + - sim: ethereum/eels/execute-blobs + limit: .*tests/osaka.* needs: - build-reth - prepare-hive @@ -375,6 +381,10 @@ jobs: limit: .*tests/frontier.* - sim: ethereum/eels/consume-rlp limit: .*tests/paris.* + + # execute-blobs + - sim: ethereum/eels/execute-blobs + limit: .*tests/osaka.* needs: - build-reth - prepare-hive From 2125dc6b23bf58fd3dffd010929956f706ba938a Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 6 Aug 2026 14:38:10 +0200 Subject: [PATCH 106/148] refactor(storage): move OverlayStateProvider and let it use a Provider ref (#26611) --- Cargo.lock | 1 + .../engine/tree/src/tree/payload_validator.rs | 10 +-- .../tree/src/tree/state_root_strategy/mod.rs | 14 ++- .../tree/state_root_strategy/sparse_trie.rs | 6 +- crates/storage/provider/src/either_writer.rs | 17 ++-- .../src/providers/blockchain_provider.rs | 8 +- .../src/providers/database/provider.rs | 21 +++-- crates/storage/provider/src/providers/mod.rs | 1 - .../provider/src/providers/state/mod.rs | 1 - .../storage/provider/src/test_utils/mock.rs | 10 ++- crates/storage/rpc-provider/src/lib.rs | 17 ++-- .../storage-api/src/database_provider.rs | 15 +++- crates/storage/storage-api/src/noop.rs | 11 ++- crates/storage/storage-overlay/Cargo.toml | 1 + crates/storage/storage-overlay/src/lib.rs | 3 + .../src/provider.rs} | 86 ++++++++----------- crates/trie/parallel/src/proof_task.rs | 2 +- 17 files changed, 123 insertions(+), 101 deletions(-) rename crates/storage/{provider/src/providers/state/overlay.rs => storage-overlay/src/provider.rs} (83%) diff --git a/Cargo.lock b/Cargo.lock index db4f92abaa0..93af0aca20a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -10456,6 +10456,7 @@ dependencies = [ "rayon", "reth-chain-state", "reth-db", + "reth-db-api", "reth-errors", "reth-ethereum-primitives", "reth-metrics", diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index b67ba7b2ac6..ea069a70fad 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -146,13 +146,13 @@ use reth_primitives_traits::{ RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable, }; use reth_provider::{ - providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockReader, ChangeSetReader, - DatabaseProviderFactory, DatabaseProviderROFactory, ProviderError, PruneCheckpointReader, - StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader, - StorageChangeSetReader, StorageSettingsCache, TryIntoHistoricalStateProvider, + BlockExecutionOutput, BlockReader, ChangeSetReader, DatabaseProviderFactory, + DatabaseProviderROFactory, ProviderError, PruneCheckpointReader, StageCheckpointReader, + StateProvider, StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader, + StorageSettingsCache, TryIntoHistoricalStateProvider, }; use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State}; -use reth_storage_overlay::OverlayManager; +use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; use reth_trie::{ hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, KeccakKeyHasher, LazyTrieData, diff --git a/crates/engine/tree/src/tree/state_root_strategy/mod.rs b/crates/engine/tree/src/tree/state_root_strategy/mod.rs index 83b3296a4ee..8edc4a0052e 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/mod.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/mod.rs @@ -69,12 +69,11 @@ use reth_primitives_traits::{ AlloyBlockHeader, FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedHeader, }; use reth_provider::{ - providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, - DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, - PruneCheckpointReader, StageCheckpointReader, StateRootProvider, StorageSettingsCache, - TryIntoHistoricalStateProvider, + BlockExecutionOutput, BlockNumReader, DatabaseProviderFactory, DatabaseProviderROFactory, + HashedPostStateProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader, + StateRootProvider, StorageSettingsCache, TryIntoHistoricalStateProvider, }; -use reth_storage_overlay::OverlayManager; +use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; use reth_tasks::utils::increase_thread_priority; use reth_trie::{ hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory, updates::TrieUpdates, @@ -1352,11 +1351,10 @@ mod tests { use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::{Account, StorageEntry}; use reth_provider::{ - providers::{BlockchainProvider, OverlayStateProviderFactory}, - test_utils::create_test_provider_factory_with_chain_spec, + providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec, HashingWriter, }; - use reth_storage_overlay::OverlayManager; + use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; use reth_testing_utils::generators; use reth_trie::test_utils::state_root; use revm::state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot, TransactionId}; diff --git a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs index 2abd518875d..9e852d9fe6a 100644 --- a/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs +++ b/crates/engine/tree/src/tree/state_root_strategy/sparse_trie.rs @@ -1121,10 +1121,8 @@ mod tests { use super::*; use alloy_primitives::{keccak256, Address, B256, U256}; use reth_db_common::init::init_genesis; - use reth_provider::{ - providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory, - }; - use reth_storage_overlay::OverlayManager; + use reth_provider::test_utils::create_test_provider_factory; + use reth_storage_overlay::{OverlayManager, OverlayStateProviderFactory}; use reth_trie_parallel::proof_task::ProofTaskCtx; use reth_trie_sparse::ArenaParallelSparseTrie; diff --git a/crates/storage/provider/src/either_writer.rs b/crates/storage/provider/src/either_writer.rs index 90f6b77ac91..b12390259e6 100644 --- a/crates/storage/provider/src/either_writer.rs +++ b/crates/storage/provider/src/either_writer.rs @@ -33,32 +33,37 @@ use reth_errors::ProviderError; use reth_node_types::NodePrimitives; use reth_primitives_traits::{ReceiptTy, StorageEntry}; use reth_static_file_types::StaticFileSegment; -use reth_storage_api::{ChangeSetReader, DBProvider, NodePrimitivesProvider, StorageSettingsCache}; +use reth_storage_api::{ + ChangeSetReader, DBProvider, DbTxProvider, NodePrimitivesProvider, StorageSettingsCache, +}; use reth_storage_errors::provider::ProviderResult; use strum::{Display, EnumIs}; /// Type alias for [`EitherReader`] constructors. -type EitherReaderTy<'a, P, T> = - EitherReader<'a, CursorTy<

::Tx, T>,

::Primitives>; +type EitherReaderTy<'a, P, T> = EitherReader< + 'a, + CursorTy<

::Tx, T>, +

::Primitives, +>; /// Type alias for [`EitherReader`] constructors. type DupEitherReaderTy<'a, P, T> = EitherReader< 'a, - DupCursorTy<

::Tx, T>, + DupCursorTy<

::Tx, T>,

::Primitives, >; /// Type alias for dup [`EitherWriter`] constructors. type DupEitherWriterTy<'a, P, T> = EitherWriter< 'a, - DupCursorMutTy<

::Tx, T>, + DupCursorMutTy<

::Tx, T>,

::Primitives, >; /// Type alias for [`EitherWriter`] constructors. type EitherWriterTy<'a, P, T> = EitherWriter< 'a, - CursorMutTy<

::Tx, T>, + CursorMutTy<

::Tx, T>,

::Primitives, >; diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index f001aca3279..77800a2d591 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -1,8 +1,8 @@ use super::state::latest::LatestStateProvider; use crate::{ providers::{ - ConsistentProvider, OverlayStateProvider, OverlayStateProviderFactory, ProviderNodeTypes, - RocksDBProvider, StaticFileProvider, StaticFileProviderRWRefMut, + ConsistentProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider, + StaticFileProviderRWRefMut, }, BalProvider, BalStoreHandle, BlockHashReader, BlockIdReader, BlockNumReader, BlockReader, BlockReaderIdExt, BlockSource, CanonChainTracker, CanonStateNotifications, @@ -36,7 +36,9 @@ use reth_storage_api::{ StorageRangeResult, TryIntoHistoricalStateProvider, }; use reth_storage_errors::provider::ProviderResult; -use reth_storage_overlay::{anchor_for_parent, AnchorForParent}; +use reth_storage_overlay::{ + anchor_for_parent, AnchorForParent, OverlayStateProvider, OverlayStateProviderFactory, +}; use reth_trie::{ hashed_cursor::{HashedCursor, HashedCursorFactory}, metrics::TrieRootMetrics, diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index 96d891e37e0..f29f84a5327 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -13,13 +13,14 @@ use crate::{ }, AccountReader, BlockBodyWriter, BlockExecutionWriter, BlockHashReader, BlockNumReader, BlockReader, BlockWriter, BundleStateInit, ChainStateBlockReader, ChainStateBlockWriter, - DBProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, HeaderProvider, - HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, HistoryWriter, - LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, PersistenceFrontiers, - ProviderError, PruneCheckpointReader, PruneCheckpointWriter, RawRocksDBBatch, RevertsInit, - RocksBatchArg, RocksDBProviderFactory, StageCheckpointReader, StateProviderBox, StateWriter, - StaticFileProviderFactory, StatsReader, StorageReader, StorageTrieWriter, TransactionVariant, - TransactionsProvider, TransactionsProviderExt, TrieWriter, + DBProvider, DbTxProvider, EitherReader, EitherWriter, EitherWriterDestination, HashingWriter, + HeaderProvider, HeaderSyncGapProvider, HistoricalStateProvider, HistoricalStateProviderRef, + HistoryWriter, LatestStateProvider, LatestStateProviderRef, OriginalValuesKnown, + PersistenceFrontiers, ProviderError, PruneCheckpointReader, PruneCheckpointWriter, + RawRocksDBBatch, RevertsInit, RocksBatchArg, RocksDBProviderFactory, StageCheckpointReader, + StateProviderBox, StateWriter, StaticFileProviderFactory, StatsReader, StorageReader, + StorageTrieWriter, TransactionVariant, TransactionsProvider, TransactionsProviderExt, + TrieWriter, }; use alloy_consensus::{ transaction::{SignerRecoverable, TransactionMeta, TxHashRef}, @@ -3980,13 +3981,15 @@ impl ChainStateBlockWriter for DatabaseProvider DBProvider for DatabaseProvider { +impl DbTxProvider for DatabaseProvider { type Tx = TX; - fn tx_ref(&self) -> &Self::Tx { + fn tx(&self) -> &Self::Tx { &self.tx } +} +impl DBProvider for DatabaseProvider { fn tx_mut(&mut self) -> &mut Self::Tx { &mut self.tx } diff --git a/crates/storage/provider/src/providers/mod.rs b/crates/storage/provider/src/providers/mod.rs index d41d335e170..ee2bc34adb5 100644 --- a/crates/storage/provider/src/providers/mod.rs +++ b/crates/storage/provider/src/providers/mod.rs @@ -20,7 +20,6 @@ pub use state::{ HistoricalStateProviderRef, HistoryInfo, LowestAvailableBlocks, }, latest::{LatestStateProvider, LatestStateProviderRef}, - overlay::{OverlayStateProvider, OverlayStateProviderFactory}, }; mod blockchain_provider; diff --git a/crates/storage/provider/src/providers/state/mod.rs b/crates/storage/provider/src/providers/state/mod.rs index fb4109fdc3e..74a1a2830e2 100644 --- a/crates/storage/provider/src/providers/state/mod.rs +++ b/crates/storage/provider/src/providers/state/mod.rs @@ -1,4 +1,3 @@ //! [`StateProvider`](crate::StateProvider) implementations pub(crate) mod historical; pub(crate) mod latest; -pub(crate) mod overlay; diff --git a/crates/storage/provider/src/test_utils/mock.rs b/crates/storage/provider/src/test_utils/mock.rs index 0a56b94748d..24264c60ffc 100644 --- a/crates/storage/provider/src/test_utils/mock.rs +++ b/crates/storage/provider/src/test_utils/mock.rs @@ -35,7 +35,7 @@ use reth_primitives_traits::{ use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment}; use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_api::{ - BlockBodyIndicesProvider, BytecodeReader, DBProvider, DatabaseProviderFactory, + BlockBodyIndicesProvider, BytecodeReader, DBProvider, DatabaseProviderFactory, DbTxProvider, HashedPostStateProvider, NodePrimitivesProvider, StageCheckpointReader, StateProofProvider, StorageChangeSetReader, StorageRootProvider, StorageSettingsCache, TryIntoHistoricalStateProvider, @@ -512,15 +512,19 @@ impl DatabaseProvi } } -impl DBProvider +impl DbTxProvider for MockEthProvider { type Tx = TxMock; - fn tx_ref(&self) -> &Self::Tx { + fn tx(&self) -> &Self::Tx { &self.tx } +} +impl DBProvider + for MockEthProvider +{ fn tx_mut(&mut self) -> &mut Self::Tx { &mut self.tx } diff --git a/crates/storage/rpc-provider/src/lib.rs b/crates/storage/rpc-provider/src/lib.rs index 73f70dd82ae..8569b55e11d 100644 --- a/crates/storage/rpc-provider/src/lib.rs +++ b/crates/storage/rpc-provider/src/lib.rs @@ -54,8 +54,8 @@ use reth_prune_types::{PruneCheckpoint, PruneSegment}; pub mod rpc_response; use reth_stages_types::{StageCheckpoint, StageId}; use reth_storage_api::{ - BlockBodyIndicesProvider, BlockReaderIdExt, BlockSource, DBProvider, NodePrimitivesProvider, - ReceiptProviderIdExt, StatsReader, + BlockBodyIndicesProvider, BlockReaderIdExt, BlockSource, DBProvider, DbTxProvider, + NodePrimitivesProvider, ReceiptProviderIdExt, StatsReader, }; use reth_trie::{ updates::TrieUpdates, AccountProof, HashedPostState, KeccakKeyHasher, MultiProof, TrieInput, @@ -1348,7 +1348,7 @@ where } } -impl DBProvider for RpcBlockchainStateProvider +impl DbTxProvider for RpcBlockchainStateProvider where P: Provider + Clone + 'static, N: Network, @@ -1356,12 +1356,19 @@ where { type Tx = TxMock; - fn tx_ref(&self) -> &Self::Tx { + fn tx(&self) -> &Self::Tx { // We can't use a static here since TxMock doesn't allow direct construction // This is fine since we're just returning a mock transaction - unimplemented!("tx_ref not supported for RPC provider") + unimplemented!("tx not supported for RPC provider") } +} +impl DBProvider for RpcBlockchainStateProvider +where + P: Provider + Clone + 'static, + N: Network, + Node: NodeTypes, +{ fn tx_mut(&mut self) -> &mut Self::Tx { unimplemented!("tx_mut not supported for RPC provider") } diff --git a/crates/storage/storage-api/src/database_provider.rs b/crates/storage/storage-api/src/database_provider.rs index 417f8e282b9..9098cc60873 100644 --- a/crates/storage/storage-api/src/database_provider.rs +++ b/crates/storage/storage-api/src/database_provider.rs @@ -11,13 +11,22 @@ use reth_db_api::{ use reth_prune_types::PruneModes; use reth_storage_errors::provider::ProviderResult; -/// Database provider. -pub trait DBProvider: Sized { +/// Provides shared access to a database transaction. +#[auto_impl::auto_impl(&)] +pub trait DbTxProvider { /// Underlying database transaction held by the provider. type Tx: DbTx; + /// Returns the underlying database transaction. + fn tx(&self) -> &Self::Tx; +} + +/// Database provider. +pub trait DBProvider: DbTxProvider + Sized { /// Returns a reference to the underlying transaction. - fn tx_ref(&self) -> &Self::Tx; + fn tx_ref(&self) -> &Self::Tx { + self.tx() + } /// Returns a mutable reference to the underlying transaction. fn tx_mut(&mut self) -> &mut Self::Tx; diff --git a/crates/storage/storage-api/src/noop.rs b/crates/storage/storage-api/src/noop.rs index 28aeb5f857e..2d8f57c3ad4 100644 --- a/crates/storage/storage-api/src/noop.rs +++ b/crates/storage/storage-api/src/noop.rs @@ -13,7 +13,9 @@ use crate::{ }; #[cfg(feature = "db-api")] -use crate::{DBProvider, DatabaseProviderFactory, StorageChangeSetReader, StorageSettingsCache}; +use crate::{ + DBProvider, DatabaseProviderFactory, DbTxProvider, StorageChangeSetReader, StorageSettingsCache, +}; use alloc::{boxed::Box, string::String, sync::Arc, vec::Vec}; use alloy_consensus::transaction::TransactionMeta; use alloy_eips::{BlockHashOrNumber, BlockId, BlockNumberOrTag}; @@ -683,13 +685,16 @@ impl BlockBodyIndicesProvider for NoopProvider DBProvider for NoopProvider { +impl DbTxProvider for NoopProvider { type Tx = TxMock; - fn tx_ref(&self) -> &Self::Tx { + fn tx(&self) -> &Self::Tx { &self.tx } +} +#[cfg(feature = "db-api")] +impl DBProvider for NoopProvider { fn tx_mut(&mut self) -> &mut Self::Tx { &mut self.tx } diff --git a/crates/storage/storage-overlay/Cargo.toml b/crates/storage/storage-overlay/Cargo.toml index f9a6e0a2b13..316ca8e8ef4 100644 --- a/crates/storage/storage-overlay/Cargo.toml +++ b/crates/storage/storage-overlay/Cargo.toml @@ -14,6 +14,7 @@ workspace = true [dependencies] # reth reth-chain-state.workspace = true +reth-db-api.workspace = true reth-errors.workspace = true reth-ethereum-primitives.workspace = true reth-metrics.workspace = true diff --git a/crates/storage/storage-overlay/src/lib.rs b/crates/storage/storage-overlay/src/lib.rs index d97f1fb67c0..426d3cc9e30 100644 --- a/crates/storage/storage-overlay/src/lib.rs +++ b/crates/storage/storage-overlay/src/lib.rs @@ -10,3 +10,6 @@ pub(crate) use changeset_cache::ChangesetCache; mod manager; pub use manager::*; + +mod provider; +pub use provider::*; diff --git a/crates/storage/provider/src/providers/state/overlay.rs b/crates/storage/storage-overlay/src/provider.rs similarity index 83% rename from crates/storage/provider/src/providers/state/overlay.rs rename to crates/storage/storage-overlay/src/provider.rs index 52316b08b27..c5f38269c0d 100644 --- a/crates/storage/provider/src/providers/state/overlay.rs +++ b/crates/storage/storage-overlay/src/provider.rs @@ -1,3 +1,4 @@ +use crate::{database_state_frontiers, Overlay, OverlayBuilder}; use alloy_primitives::{BlockHash, B256}; use metrics::{Counter, Histogram}; use reth_db_api::{tables, transaction::DbTx, DatabaseError}; @@ -10,14 +11,12 @@ use reth_primitives_traits::{ }; use reth_storage_api::{ BlockNumReader, ChangeSetReader, DBProvider, DatabaseProviderFactory, - DatabaseProviderROFactory, PruneCheckpointReader, StageCheckpointReader, + DatabaseProviderROFactory, DbTxProvider, PruneCheckpointReader, StageCheckpointReader, StorageChangeSetReader, StorageSettingsCache, }; -use reth_storage_overlay::{database_state_frontiers, Overlay, OverlayBuilder}; use reth_trie::{ hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory}, trie_cursor::{InMemoryTrieCursor, TrieCursor, TrieCursorFactory, TrieStorageCursor}, - updates::TrieUpdatesSorted, HashedPostStateSorted, }; use reth_trie_db::{ @@ -118,7 +117,6 @@ where F: DatabaseProviderFactory, F::Provider: StageCheckpointReader + PruneCheckpointReader - + DBProvider + BlockNumReader + ChangeSetReader + StorageChangeSetReader @@ -139,11 +137,11 @@ where res }; - let Overlay { trie_updates, hashed_post_state } = self.get_overlay(&provider)?; + let overlay = self.get_overlay(&provider)?; let is_v2 = provider.cached_storage_settings().is_v2(); self.metrics.database_provider_ro_duration.record(overall_start.elapsed()); - Ok(OverlayStateProvider::new(provider, trie_updates, hashed_post_state, is_v2)) + Ok(OverlayStateProvider::new(provider, overlay, is_v2)) } } @@ -153,33 +151,22 @@ where /// on top of a database provider, implementing [`TrieCursorFactory`] and [`HashedCursorFactory`] /// using the in-memory overlay factories. #[derive(Debug)] -pub struct OverlayStateProvider { +pub struct OverlayStateProvider { provider: Provider, - trie_updates: Arc, - hashed_post_state: Arc, + overlay: Overlay, is_v2: bool, } -impl OverlayStateProvider -where - Provider: DBProvider, -{ - /// Create new overlay state provider. The `Provider` must be cloneable, which generally means - /// it should be wrapped in an `Arc`. - pub const fn new( - provider: Provider, - trie_updates: Arc, - hashed_post_state: Arc, - is_v2: bool, - ) -> Self { - Self { provider, trie_updates, hashed_post_state, is_v2 } +impl OverlayStateProvider { + /// Creates a new overlay state provider. + pub const fn new(provider: Provider, overlay: Overlay, is_v2: bool) -> Self { + Self { provider, overlay, is_v2 } } } impl TrieCursorFactory for OverlayStateProvider where - Provider: DBProvider, - Provider::Tx: DbTx, + Provider: DbTxProvider, { type AccountTrieCursor<'a> = InMemoryTrieCursor<'a, Box> @@ -192,44 +179,40 @@ where Self: 'a; fn account_trie_cursor(&self) -> Result, DatabaseError> { - let tx = self.provider.tx_ref(); - let trie_updates = self.trie_updates.as_ref(); let cursor: Box = if self.is_v2 { Box::new(DatabaseAccountTrieCursor::<_, PackedKeyAdapter>::new( - tx.cursor_read::()?, + self.provider.tx().cursor_read::()?, )) } else { Box::new(DatabaseAccountTrieCursor::<_, LegacyKeyAdapter>::new( - tx.cursor_read::()?, + self.provider.tx().cursor_read::()?, )) }; - Ok(InMemoryTrieCursor::new_account(cursor, trie_updates)) + Ok(InMemoryTrieCursor::new_account(cursor, &self.overlay.trie_updates)) } fn storage_trie_cursor( &self, hashed_address: B256, ) -> Result, DatabaseError> { - let tx = self.provider.tx_ref(); - let trie_updates = self.trie_updates.as_ref(); let cursor: Box = if self.is_v2 { Box::new(DatabaseStorageTrieCursor::<_, PackedKeyAdapter>::new( - tx.cursor_dup_read::()?, + self.provider.tx().cursor_dup_read::()?, hashed_address, )) } else { Box::new(DatabaseStorageTrieCursor::<_, LegacyKeyAdapter>::new( - tx.cursor_dup_read::()?, + self.provider.tx().cursor_dup_read::()?, hashed_address, )) }; - Ok(InMemoryTrieCursor::new_storage(cursor, trie_updates, hashed_address)) + Ok(InMemoryTrieCursor::new_storage(cursor, &self.overlay.trie_updates, hashed_address)) } } impl HashedCursorFactory for OverlayStateProvider where - Provider: DBProvider, + Provider: DbTxProvider, { type AccountCursor<'a> = Result, DatabaseError> { - let db_hashed_cursor_factory = DatabaseHashedCursorFactory::new(self.provider.tx_ref()); - let hashed_cursor_factory = - HashedPostStateCursorFactory::new(db_hashed_cursor_factory, &self.hashed_post_state); - hashed_cursor_factory.hashed_account_cursor() + HashedPostStateCursorFactory::new( + DatabaseHashedCursorFactory::new(self.provider.tx()), + &self.overlay.hashed_post_state, + ) + .hashed_account_cursor() } fn hashed_storage_cursor( &self, hashed_address: B256, ) -> Result, DatabaseError> { - let db_hashed_cursor_factory = DatabaseHashedCursorFactory::new(self.provider.tx_ref()); - let hashed_cursor_factory = - HashedPostStateCursorFactory::new(db_hashed_cursor_factory, &self.hashed_post_state); - hashed_cursor_factory.hashed_storage_cursor(hashed_address) + HashedPostStateCursorFactory::new( + DatabaseHashedCursorFactory::new(self.provider.tx()), + &self.overlay.hashed_post_state, + ) + .hashed_storage_cursor(hashed_address) } } #[cfg(all(test, feature = "partial-persistence"))] mod tests { use super::*; - use crate::{ - test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, - BlockWriter, ProviderFactory, - }; + use crate::OverlayManager; use alloy_primitives::U256; use reth_chain_state::{test_utils::TestBlockBuilder, ExecutedBlock}; use reth_primitives_traits::Account; + use reth_provider::{ + test_utils::{create_test_provider_factory, MockNodeTypesWithDB}, + BlockWriter, ProviderFactory, + }; use reth_stages_types::{FinishCheckpoint, StageCheckpoint, StageId}; use reth_storage_api::StageCheckpointWriter; - use reth_storage_overlay::OverlayManager; - use reth_trie::{BranchNodeCompact, ComputedTrieData, HashedPostState, HashedStorage, Nibbles}; + use reth_trie::{ + updates::TrieUpdatesSorted, BranchNodeCompact, ComputedTrieData, HashedPostState, + HashedStorage, Nibbles, + }; fn with_unique_trie_data( block: &ExecutedBlock, diff --git a/crates/trie/parallel/src/proof_task.rs b/crates/trie/parallel/src/proof_task.rs index 3f98c01dcf1..1e77d654fac 100644 --- a/crates/trie/parallel/src/proof_task.rs +++ b/crates/trie/parallel/src/proof_task.rs @@ -1190,7 +1190,7 @@ mod tests { let chain_spec = Arc::new(ChainSpec::default()); let anchor_hash = chain_spec.genesis_hash(); let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec); - let factory = reth_provider::providers::OverlayStateProviderFactory::new( + let factory = reth_storage_overlay::OverlayStateProviderFactory::new( provider_factory, reth_storage_overlay::OverlayManager::< reth_ethereum_primitives::EthPrimitives, From b8f775a90f3543f69fe12b299603cfdc040d20be Mon Sep 17 00:00:00 2001 From: Brian Picciano Date: Thu, 6 Aug 2026 15:00:51 +0200 Subject: [PATCH 107/148] fix(trie): support partial persistence in changeset cache (#26612) --- crates/storage/storage-overlay/src/builder.rs | 44 +++++- .../storage-overlay/src/changeset_cache.rs | 141 ++++++++++++++---- crates/storage/storage-overlay/src/manager.rs | 45 +++++- crates/trie/db/src/changesets.rs | 111 ++++++-------- crates/trie/db/src/lib.rs | 2 +- crates/trie/db/src/state.rs | 20 +-- 6 files changed, 236 insertions(+), 127 deletions(-) diff --git a/crates/storage/storage-overlay/src/builder.rs b/crates/storage/storage-overlay/src/builder.rs index 7096e1681c5..2452da5cf30 100644 --- a/crates/storage/storage-overlay/src/builder.rs +++ b/crates/storage/storage-overlay/src/builder.rs @@ -14,6 +14,7 @@ use reth_storage_api::{ StorageChangeSetReader, StorageSettingsCache, }; use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted}; +use reth_trie_db::DatabaseHashedPostState; use std::{ sync::Arc, time::{Duration, Instant}, @@ -69,6 +70,8 @@ pub struct OverlayBuilder { overlay_manager: OverlayManager, /// Anchor hash of the reused sparse trie, if this task reused one. reused_sparse_trie_anchor_hash: Option, + /// Whether building the overlay may query revert changesets. + no_reverts: bool, /// Metrics for overlay construction. metrics: OverlayBuilderMetrics, } @@ -81,6 +84,7 @@ impl OverlayBuilder { overlay_source: Some(OverlaySource::Managed), overlay_manager, reused_sparse_trie_anchor_hash: None, + no_reverts: false, metrics: OverlayBuilderMetrics::default(), } } @@ -100,6 +104,12 @@ impl OverlayBuilder { self } + /// Returns an error instead of querying revert changesets when reverts are required. + pub const fn with_no_reverts(mut self) -> Self { + self.no_reverts = true; + self + } + /// Sets an immediate hashed-state and trie-updates overlay. pub fn with_immediate_state_trie_overlay( mut self, @@ -200,6 +210,13 @@ impl OverlayBuilder { // Collect any reverts which are required to bring the DB view back to the anchor hash. let (trie_updates, hashed_post_state) = match anchor_for_parent { AnchorForParent::RevertsRequired { anchor, finish, .. } => { + if self.no_reverts { + return Err(ProviderError::other(std::io::Error::other(format!( + "reverts are disabled, but overlay for parent {} requires reverting Finish #{} ({}) to anchor #{} ({})", + self.parent_hash, finish.number, finish.hash, anchor.number, anchor.hash, + )))) + } + let revert_blocks = anchor.number + 1..=finish.number; debug!( @@ -213,9 +230,13 @@ impl OverlayBuilder { let _guard = debug_span!(target: "storage::overlay", "retrieving_trie_reverts") .entered(); let start = Instant::now(); - let accumulated_reverts = self - .overlay_manager - .get_or_compute_cached_changesets_range(provider, revert_blocks.clone())?; + let accumulated_reverts = + self.overlay_manager.get_or_compute_cached_changesets_range_at_frontiers( + provider, + revert_blocks.clone(), + state_trie_tip_block, + finish_tip_block, + )?; retrieve_trie_reverts_duration = start.elapsed(); accumulated_reverts }; @@ -225,7 +246,7 @@ impl OverlayBuilder { debug_span!(target: "storage::overlay", "retrieving_hashed_state_reverts") .entered(); let start = Instant::now(); - let res = reth_trie_db::from_reverts_auto(provider, revert_blocks)?; + let res = HashedPostStateSorted::from_reverts(provider, revert_blocks)?; retrieve_hashed_state_reverts_duration = start.elapsed(); res }; @@ -721,6 +742,21 @@ mod tests { assert!(overlay.trie_updates.is_empty()); } + #[cfg(feature = "partial-persistence")] + #[test] + fn no_reverts_errors_when_reverts_are_required() { + let (factory, blocks) = setup_frontiers(2, 3); + let provider = factory.provider().unwrap(); + + let error = OverlayManager::::default() + .overlay_builder(blocks[1].recovered_block().hash()) + .with_no_reverts() + .build_overlay(&provider) + .unwrap_err(); + + assert!(error.to_string().contains("reverts are disabled")); + } + #[cfg(feature = "partial-persistence")] #[test] fn managed_overlay_uses_persisted_parent_even_if_retained() { diff --git a/crates/storage/storage-overlay/src/changeset_cache.rs b/crates/storage/storage-overlay/src/changeset_cache.rs index bc6ace93130..1400a432579 100644 --- a/crates/storage/storage-overlay/src/changeset_cache.rs +++ b/crates/storage/storage-overlay/src/changeset_cache.rs @@ -7,15 +7,18 @@ //! - **Reorg support**: Quickly access changesets to revert blocks during chain reorganizations //! - **Memory efficiency**: Explicit eviction releases persisted changesets +use crate::{database_state_frontiers, OverlayManager, OverlayStateProvider}; +use alloy_eips::BlockNumHash; use alloy_primitives::{map::B256Map, BlockNumber, B256}; use parking_lot::RwLock; use reth_metrics::{ metrics::{Counter, Gauge}, Metrics, }; -use reth_primitives_traits::FastInstant as Instant; +use reth_primitives_traits::{FastInstant as Instant, NodePrimitives}; use reth_storage_api::{ - BlockNumReader, ChangeSetReader, DBProvider, StorageChangeSetReader, StorageSettingsCache, + BlockNumReader, ChangeSetReader, DBProvider, PruneCheckpointReader, StageCheckpointReader, + StorageChangeSetReader, StorageSettingsCache, }; use reth_storage_errors::provider::{ProviderError, ProviderResult}; use reth_trie::trie_cursor::{InMemoryTrieCursorFactory, TrieCursor, TrieCursorFactory}; @@ -29,9 +32,9 @@ use std::{ use tracing::{debug, warn}; #[cfg(test)] -use reth_trie::{changesets::compute_trie_changesets, TrieInputSorted}; +use reth_trie::{changesets::compute_trie_changesets, HashedPostStateSorted, TrieInputSorted}; #[cfg(test)] -use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; +use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseHashedPostState, DatabaseStateRoot}; /// Computes block trie updates using the changeset cache. /// @@ -48,7 +51,6 @@ use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; /// /// # Arguments /// -/// * `cache` - Handle to the changeset cache for retrieving trie reverts /// * `provider` - Database provider for accessing changesets and block data /// * `block_number` - Block number to compute trie updates for /// @@ -62,45 +64,63 @@ use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseStateRoot}; /// - Block number exceeds database tip /// - Database access fails /// - Cache retrieval fails -pub(crate) fn compute_block_trie_updates( - cache: &ChangesetCache, +pub(crate) fn compute_block_trie_updates( + overlay_manager: &OverlayManager, provider: &Provider, block_number: BlockNumber, ) -> ProviderResult where + N: NodePrimitives, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + + PruneCheckpointReader + + StageCheckpointReader + BlockNumReader + StorageSettingsCache, { reth_trie_db::with_adapter!(provider, |A| { - compute_block_trie_updates_inner::<_, A>(cache, provider, block_number) + compute_block_trie_updates_inner::<_, _, A>(overlay_manager, provider, block_number) }) } -fn compute_block_trie_updates_inner( - cache: &ChangesetCache, +fn compute_block_trie_updates_inner( + overlay_manager: &OverlayManager, provider: &Provider, block_number: BlockNumber, ) -> ProviderResult where + N: NodePrimitives, Provider: DBProvider + ChangeSetReader + StorageChangeSetReader + + PruneCheckpointReader + + StageCheckpointReader + BlockNumReader + StorageSettingsCache, A: TrieTableAdapter, { let tx = provider.tx_ref(); - - let db_tip_block = provider.best_block_number()?; + let cache = overlay_manager.changeset_cache(); + let (partial_state_trie, finish) = database_state_frontiers(provider)?; // Step 1: Get the trie changesets for the target block from cache - let changesets = cache.get_or_compute(provider, block_number)?; + let changesets = cache.get_or_compute( + overlay_manager, + provider, + block_number, + partial_state_trie, + finish, + )?; // Step 2: Get the trie reverts for the state after the target block using the cache - let reverts = cache.get_or_compute_range(provider, (block_number + 1)..=db_tip_block)?; + let reverts = cache.get_or_compute_range( + overlay_manager, + provider, + (block_number + 1)..=finish.number, + partial_state_trie, + finish, + )?; // Step 3: Create an InMemoryTrieCursorFactory with the reverts // This gives us the trie state as it was after the target block was processed @@ -194,19 +214,30 @@ impl ChangesetCache { /// # Returns /// /// Changesets for the block, either from cache or computed on-the-fly. - pub(crate) fn get_or_compute

( + pub(crate) fn get_or_compute( &self, + overlay_manager: &OverlayManager, provider: &P, block_number: BlockNumber, + partial_state_trie: BlockNumHash, + finish: BlockNumHash, ) -> ProviderResult> where + N: NodePrimitives, P: DBProvider + ChangeSetReader + StorageChangeSetReader + + PruneCheckpointReader + BlockNumReader + StorageSettingsCache, { - self.get_or_compute_range(provider, block_number..=block_number) + self.get_or_compute_range( + overlay_manager, + provider, + block_number..=block_number, + partial_state_trie, + finish, + ) } /// Gets or computes trie reverts for a range of blocks. @@ -235,38 +266,40 @@ impl ChangesetCache { /// - Database access fails /// - Block hash lookup fails /// - Changeset computation fails - pub(crate) fn get_or_compute_range

( + pub(crate) fn get_or_compute_range( &self, + overlay_manager: &OverlayManager, provider: &P, range: RangeInclusive, + partial_state_trie: BlockNumHash, + finish: BlockNumHash, ) -> ProviderResult> where + N: NodePrimitives, P: DBProvider + ChangeSetReader + StorageChangeSetReader + + PruneCheckpointReader + BlockNumReader + StorageSettingsCache, { - let db_tip_block = provider.best_block_number()?; - let start_block = *range.start(); let end_block = *range.end(); + let timer = Instant::now(); - // If range end is beyond the tip, return an error - if end_block > db_tip_block { + if end_block > finish.number { return Err(ProviderError::InsufficientChangesets { requested: end_block, - available: 0..=db_tip_block, + available: 0..=finish.number, }); } - let timer = Instant::now(); - debug!( target: "trie::changeset_cache", start_block, end_block, - db_tip_block, + ?partial_state_trie, + ?finish, "Starting get_or_compute_range" ); @@ -368,10 +401,21 @@ impl ChangesetCache { "Changeset cache MISS in range, falling back to aggregate DB-based computation" ); + let overlay = overlay_manager + .overlay_builder(finish.hash) + .with_no_reverts() + .build_overlay_at_frontiers(provider, partial_state_trie, finish)?; + let state_trie_provider = OverlayStateProvider::new( + provider, + overlay, + provider.cached_storage_settings().is_v2(), + ); + let accumulated_reverts = Arc::new(reth_trie_db::compute_range_trie_changesets( provider, + &state_trie_provider, start_block..=end_block, - db_tip_block, + finish.number, )?); let elapsed = timer.elapsed(); @@ -574,6 +618,7 @@ impl ChangesetCacheInner { #[cfg(test)] mod tests { use super::*; + use crate::Overlay; use alloy_consensus::Header; use alloy_primitives::{ keccak256, @@ -599,6 +644,10 @@ mod tests { Arc::new(TrieUpdatesSorted::new(vec![], B256Map::default())) } + fn empty_overlay() -> Overlay { + Overlay { trie_updates: Arc::default(), hashed_post_state: Arc::default() } + } + fn insert_test_changesets( cache: &mut ChangesetCacheInner, block_hash: B256, @@ -690,9 +739,9 @@ mod tests { A: TrieTableAdapter, { let individual_state_revert = - reth_trie_db::from_reverts_auto(provider, block_number..=block_number).unwrap(); + HashedPostStateSorted::from_reverts(provider, block_number..=block_number).unwrap(); let cumulative_state_revert = - reth_trie_db::from_reverts_auto(provider, (block_number + 1)..).unwrap(); + HashedPostStateSorted::from_reverts(provider, (block_number + 1)..).unwrap(); let mut cumulative_state_revert_prev = cumulative_state_revert.clone(); cumulative_state_revert_prev.extend_ref_and_sort(&individual_state_revert); @@ -780,7 +829,11 @@ mod tests { ); } - let accumulated = cache.get_or_compute_range(&*provider, 1..=2).unwrap(); + let overlay_manager = OverlayManager::::default(); + let (partial_state_trie, finish) = database_state_frontiers(&*provider).unwrap(); + let accumulated = cache + .get_or_compute_range(&overlay_manager, &*provider, 1..=2, partial_state_trie, finish) + .unwrap(); assert_eq!(accumulated.account_nodes_ref(), &[(path, Some(older_node))]); } @@ -853,7 +906,15 @@ mod tests { provider.save_stage_checkpoint(StageId::Finish, StageCheckpoint::new(3)).unwrap(); reth_trie_db::with_adapter!(provider, |A| seed_tip_trie_tables::<_, A>(&*provider)); - let actual = reth_trie_db::compute_range_trie_changesets(&*provider, 1..=3, 3).unwrap(); + let overlay = empty_overlay(); + let state_trie_provider = OverlayStateProvider::new( + &*provider, + overlay, + provider.cached_storage_settings().is_v2(), + ); + let actual = + reth_trie_db::compute_range_trie_changesets(&*provider, &state_trie_provider, 1..=3, 3) + .unwrap(); let storage_revert = actual .storage_tries_ref() .get(&hashed_address) @@ -862,11 +923,17 @@ mod tests { assert!(storage_revert.storage_nodes_ref().is_empty()); let cache = ChangesetCache::new(); - let from_cache_api = cache.get_or_compute_range(&*provider, 1..=3).unwrap(); + let overlay_manager = OverlayManager::::default(); + let (partial_state_trie, finish) = database_state_frontiers(&*provider).unwrap(); + let from_cache_api = cache + .get_or_compute_range(&overlay_manager, &*provider, 1..=3, partial_state_trie, finish) + .unwrap(); assert_eq!(*from_cache_api, actual); assert_eq!(cache.inner.read().entries.len(), 1); - let block_changesets = cache.get_or_compute(&*provider, 2).unwrap(); + let block_changesets = cache + .get_or_compute(&overlay_manager, &*provider, 2, partial_state_trie, finish) + .unwrap(); assert_eq!(*block_changesets, legacy_compute_block_trie_changesets(&*provider, 2)); assert_eq!(cache.inner.read().entries.len(), 2); } @@ -930,7 +997,15 @@ mod tests { reth_trie_db::with_adapter!(provider, |A| seed_tip_trie_tables::<_, A>(&*provider)); let expected = legacy_compute_range_trie_changesets(&*provider, 2..=3); - let actual = reth_trie_db::compute_range_trie_changesets(&*provider, 2..=3, 3).unwrap(); + let overlay = empty_overlay(); + let state_trie_provider = OverlayStateProvider::new( + &*provider, + overlay, + provider.cached_storage_settings().is_v2(), + ); + let actual = + reth_trie_db::compute_range_trie_changesets(&*provider, &state_trie_provider, 2..=3, 3) + .unwrap(); assert_eq!(actual, expected); } diff --git a/crates/storage/storage-overlay/src/manager.rs b/crates/storage/storage-overlay/src/manager.rs index 7f677b3df8b..466d9625703 100644 --- a/crates/storage/storage-overlay/src/manager.rs +++ b/crates/storage/storage-overlay/src/manager.rs @@ -4,7 +4,11 @@ //! parent has not been persisted yet. [`OverlayManager`] tracks those in-memory blocks and //! builds reusable flattened state trie overlays on demand. -use crate::{changeset_cache::compute_block_trie_updates, ChangesetCache, OverlayBuilder}; +use crate::{ + changeset_cache::compute_block_trie_updates, database_state_frontiers, ChangesetCache, + OverlayBuilder, +}; +use alloy_eips::BlockNumHash; use alloy_primitives::{BlockNumber, B256}; use parking_lot::Mutex; use reth_chain_state::{ExecutedBlock, PreservedSparseTrie}; @@ -19,7 +23,8 @@ use reth_primitives_traits::{ AlloyBlockHeader, FastInstant, NodePrimitives, }; use reth_storage_api::{ - BlockNumReader, ChangeSetReader, DBProvider, StorageChangeSetReader, StorageSettingsCache, + BlockNumReader, ChangeSetReader, DBProvider, PruneCheckpointReader, StageCheckpointReader, + StorageChangeSetReader, StorageSettingsCache, }; #[cfg(feature = "rayon")] use reth_tasks::WorkerPool; @@ -101,6 +106,10 @@ impl OverlayManager { OverlayBuilder::new(parent_hash, self.clone()) } + pub(crate) const fn changeset_cache(&self) -> &ChangesetCache { + &self.changeset_cache + } + /// Gets or computes cached changesets for an inclusive block range. pub fn get_or_compute_cached_changesets_range

( &self, @@ -111,10 +120,36 @@ impl OverlayManager { P: DBProvider + ChangeSetReader + StorageChangeSetReader + + PruneCheckpointReader + + StageCheckpointReader + + BlockNumReader + + StorageSettingsCache, + { + let (partial_state_trie, finish) = database_state_frontiers(provider)?; + self.get_or_compute_cached_changesets_range_at_frontiers( + provider, + range, + partial_state_trie, + finish, + ) + } + + pub(crate) fn get_or_compute_cached_changesets_range_at_frontiers

( + &self, + provider: &P, + range: RangeInclusive, + partial_state_trie: BlockNumHash, + finish: BlockNumHash, + ) -> ProviderResult> + where + P: DBProvider + + ChangeSetReader + + StorageChangeSetReader + + PruneCheckpointReader + BlockNumReader + StorageSettingsCache, { - self.changeset_cache.get_or_compute_range(provider, range) + self.changeset_cache.get_or_compute_range(self, provider, range, partial_state_trie, finish) } /// Evicts cached changesets for blocks below `up_to_block`. @@ -132,10 +167,12 @@ impl OverlayManager { P: DBProvider + ChangeSetReader + StorageChangeSetReader + + PruneCheckpointReader + + StageCheckpointReader + BlockNumReader + StorageSettingsCache, { - compute_block_trie_updates(&self.changeset_cache, provider, block_number) + compute_block_trie_updates(self, provider, block_number) } /// Takes the preserved sparse trie if present. diff --git a/crates/trie/db/src/changesets.rs b/crates/trie/db/src/changesets.rs index 80a2805846f..4ba454b1d8f 100644 --- a/crates/trie/db/src/changesets.rs +++ b/crates/trie/db/src/changesets.rs @@ -3,17 +3,17 @@ //! This module reconstructs trie changesets from database state. The resulting changesets contain //! the old trie node values needed to revert a block or contiguous range of blocks. -use crate::{ - DatabaseHashedCursorFactory, DatabaseStateRoot, DatabaseTrieCursorFactory, TrieTableAdapter, -}; +use crate::DatabaseHashedPostState; use alloy_primitives::BlockNumber; -use reth_storage_api::{ - BlockNumReader, ChangeSetReader, DBProvider, StorageChangeSetReader, StorageSettingsCache, -}; +use reth_storage_api::{BlockNumReader, ChangeSetReader, StorageChangeSetReader}; use reth_storage_errors::provider::ProviderError; -use reth_trie::TrieInputSorted; +use reth_trie::{ + hashed_cursor::{HashedCursorFactory, HashedPostStateCursorFactory}, + trie_cursor::{InMemoryTrieCursorFactory, TrieCursorFactory}, + StateRoot, +}; use reth_trie_common::updates::TrieUpdatesSorted; -use std::{ops::RangeInclusive, sync::Arc}; +use std::ops::RangeInclusive; use tracing::debug; /// Computes trie changesets for a block. @@ -25,25 +25,22 @@ use tracing::debug; /// /// Returns an error if the block exceeds the database tip, database access fails, or state root /// computation fails. -pub fn compute_block_trie_changesets( +pub fn compute_block_trie_changesets( provider: &Provider, + state_trie_provider: &StateTrieProvider, block_number: BlockNumber, ) -> Result where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, + Provider: ChangeSetReader + StorageChangeSetReader + BlockNumReader, + StateTrieProvider: TrieCursorFactory + HashedCursorFactory, { let db_tip_block = provider.best_block_number()?; - crate::with_adapter!(provider, |A| { - compute_range_trie_changesets_inner::<_, A>( - provider, - block_number..=block_number, - db_tip_block, - ) - }) + compute_range_trie_changesets( + provider, + state_trie_provider, + block_number..=block_number, + db_tip_block, + ) } /// Computes aggregate trie changesets for an inclusive block range. @@ -55,35 +52,15 @@ where /// /// Returns an error if the range exceeds `db_tip_block`, database access fails, or state root /// computation fails. -pub fn compute_range_trie_changesets( - provider: &Provider, - range: RangeInclusive, - db_tip_block: BlockNumber, -) -> Result -where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, -{ - crate::with_adapter!(provider, |A| { - compute_range_trie_changesets_inner::<_, A>(provider, range, db_tip_block) - }) -} - -fn compute_range_trie_changesets_inner( +pub fn compute_range_trie_changesets( provider: &Provider, + state_trie_provider: &StateTrieProvider, range: RangeInclusive, db_tip_block: BlockNumber, ) -> Result where - Provider: DBProvider - + ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + StorageSettingsCache, - A: TrieTableAdapter, + Provider: ChangeSetReader + StorageChangeSetReader + BlockNumReader, + StateTrieProvider: TrieCursorFactory + HashedCursorFactory, { let start_block = *range.start(); let end_block = *range.end(); @@ -108,14 +85,9 @@ where ); // Collect the state revert for the requested range. - let range_state_revert = crate::state::from_reverts_auto(provider, range)?; + let range_state_revert = reth_trie::HashedPostStateSorted::from_reverts(provider, range)?; let range_prefix_sets = range_state_revert.construct_prefix_sets(); - type DbStateRoot<'a, TX, A> = reth_trie::StateRoot< - DatabaseTrieCursorFactory<&'a TX, A>, - DatabaseHashedCursorFactory<&'a TX>, - >; - let (range_nodes, range_state) = if end_block == db_tip_block { debug!( target: "trie::changesets", @@ -125,25 +97,25 @@ where "Skipping tail trie revert computation for tip-ended range" ); - (Arc::default(), Arc::new(range_state_revert)) + (TrieUpdatesSorted::default(), range_state_revert) } else { // Collect the state revert from the database tip to just after the range. let tail_state_revert = end_block .checked_add(1) - .map(|next_block| crate::state::from_reverts_auto(provider, next_block..)) + .map(|next_block| { + reth_trie::HashedPostStateSorted::from_reverts(provider, next_block..) + }) .transpose()? .unwrap_or_default(); // Compute trie reverts from the database tip to just after the range. - let tail_input = TrieInputSorted::new( - Arc::default(), - Arc::new(tail_state_revert.clone()), - tail_state_revert.construct_prefix_sets(), - ); - let tail_trie_revert = DbStateRoot::<_, A>::overlay_root_from_nodes_with_updates( - provider.tx_ref(), - tail_input, + let tail_prefix_sets = tail_state_revert.construct_prefix_sets().freeze(); + let tail_trie_revert = StateRoot::new( + state_trie_provider, + HashedPostStateCursorFactory::new(state_trie_provider, &tail_state_revert), ) + .with_prefix_sets(tail_prefix_sets) + .root_with_updates() .map_err(ProviderError::other)? .1 .into_sorted(); @@ -152,15 +124,18 @@ where let mut pre_range_state_revert = tail_state_revert; pre_range_state_revert.extend_ref_and_sort(&range_state_revert); - (Arc::new(tail_trie_revert), Arc::new(pre_range_state_revert)) + (tail_trie_revert, pre_range_state_revert) }; - let range_input = TrieInputSorted::new(range_nodes, range_state, range_prefix_sets); - let range_trie_revert = - DbStateRoot::<_, A>::overlay_root_from_nodes_with_updates(provider.tx_ref(), range_input) - .map_err(ProviderError::other)? - .1 - .into_sorted(); + let range_trie_revert = StateRoot::new( + InMemoryTrieCursorFactory::new(state_trie_provider, &range_nodes), + HashedPostStateCursorFactory::new(state_trie_provider, &range_state), + ) + .with_prefix_sets(range_prefix_sets.freeze()) + .root_with_updates() + .map_err(ProviderError::other)? + .1 + .into_sorted(); debug!( target: "trie::changesets", diff --git a/crates/trie/db/src/lib.rs b/crates/trie/db/src/lib.rs index 31bede063fd..f9b44133fe7 100644 --- a/crates/trie/db/src/lib.rs +++ b/crates/trie/db/src/lib.rs @@ -17,7 +17,7 @@ pub use hashed_cursor::{ pub use prefix_set::load_prefix_sets_with_provider; pub use proof::{DatabaseProof, DatabaseStorageProof}; pub use reth_db_api::tables::{PackedAccountsTrie, PackedStoragesTrie}; -pub use state::{from_reverts_auto, DatabaseHashedPostState, DatabaseStateRoot}; +pub use state::{DatabaseHashedPostState, DatabaseStateRoot}; pub use storage::{hashed_storage_from_reverts_with_provider, DatabaseStorageRoot}; pub use trie_cursor::{ DatabaseAccountTrieCursor, DatabaseStorageTrieCursor, DatabaseTrieCursorFactory, diff --git a/crates/trie/db/src/state.rs b/crates/trie/db/src/state.rs index 216ffab1fb2..c0012fdae05 100644 --- a/crates/trie/db/src/state.rs +++ b/crates/trie/db/src/state.rs @@ -5,9 +5,7 @@ use reth_db_api::{ transaction::DbTx, }; use reth_execution_errors::StateRootError; -use reth_storage_api::{ - BlockNumReader, ChangeSetReader, DBProvider, StorageChangeSetReader, StorageSettingsCache, -}; +use reth_storage_api::{ChangeSetReader, DBProvider, StorageChangeSetReader, StorageSettingsCache}; use reth_storage_errors::provider::ProviderError; use reth_trie::{ hashed_cursor::HashedPostStateCursorFactory, trie_cursor::InMemoryTrieCursorFactory, @@ -146,7 +144,7 @@ pub trait DatabaseHashedPostState: Sized { /// Initializes [`HashedPostStateSorted`] from reverts. Iterates over state reverts in the /// specified range and aggregates them into sorted hashed state. fn from_reverts( - provider: &(impl ChangeSetReader + StorageChangeSetReader + BlockNumReader + DBProvider), + provider: &(impl ChangeSetReader + StorageChangeSetReader), range: impl RangeBounds, ) -> Result; } @@ -263,18 +261,6 @@ impl<'a, TX: DbTx, A: crate::TrieTableAdapter> DatabaseStateRoot<'a, TX> } } -/// Calls [`HashedPostStateSorted::from_reverts`]. -pub fn from_reverts_auto( - provider: &(impl ChangeSetReader - + StorageChangeSetReader - + BlockNumReader - + DBProvider - + StorageSettingsCache), - range: impl RangeBounds, -) -> Result { - HashedPostStateSorted::from_reverts(provider, range) -} - impl DatabaseHashedPostState for HashedPostStateSorted { /// Builds a sorted hashed post-state from reverts. /// @@ -287,7 +273,7 @@ impl DatabaseHashedPostState for HashedPostStateSorted { /// - Returns keys already ordered for trie iteration. #[instrument(target = "trie::db", skip(provider), fields(range))] fn from_reverts( - provider: &(impl ChangeSetReader + StorageChangeSetReader + BlockNumReader + DBProvider), + provider: &(impl ChangeSetReader + StorageChangeSetReader), range: impl RangeBounds, ) -> Result { // Extract concrete start/end values to use for both account and storage changesets. From 9253c0f8bc62469e17abb9b5b122f6fc4ae27ad2 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Thu, 6 Aug 2026 19:10:32 +0200 Subject: [PATCH 108/148] fix(dns): skip unrelated TXT records (#26603) --- crates/net/dns/src/resolver.rs | 40 ++++++++++++++++++++++++++-------- crates/net/dns/src/tree.rs | 7 ++++++ 2 files changed, 38 insertions(+), 9 deletions(-) diff --git a/crates/net/dns/src/resolver.rs b/crates/net/dns/src/resolver.rs index 3deff398a6b..4b14d5d597c 100644 --- a/crates/net/dns/src/resolver.rs +++ b/crates/net/dns/src/resolver.rs @@ -1,9 +1,10 @@ //! Perform DNS lookups +use crate::tree::has_entry_prefix; use dashmap::DashMap; pub use hickory_resolver::{net::NetError, TokioResolver}; use hickory_resolver::{ - proto::rr::{rdata::TXT, RData}, + proto::rr::{rdata::TXT, RData, Record}, ConnectionProvider, }; use std::future::Future; @@ -25,17 +26,22 @@ impl Resolver for hickory_resolver::Resolver

{ trace!(target: "disc::dns", %err, ?query, "dns lookup failed"); None } - Ok(lookup) => { - let txt = lookup.answers().iter().find_map(|r| match &r.data { - RData::TXT(txt) => Some(txt), - _ => None, - })?; - txt_entry(txt) - } + Ok(lookup) => find_txt_entry(lookup.answers()), } } } +/// Returns the first TXT record with a recognized EIP-1459 entry prefix. +fn find_txt_entry(records: &[Record]) -> Option { + records + .iter() + .filter_map(|record| { + let RData::TXT(txt) = &record.data else { return None }; + txt_entry(txt) + }) + .find(|entry| has_entry_prefix(entry)) +} + /// Joins all ``s of a TXT record into a single entry. /// /// [RFC 1035](https://www.rfc-editor.org/rfc/rfc1035#section-3.3) limits a single @@ -138,7 +144,7 @@ mod tests { net::runtime::TokioRuntimeProvider, proto::{ op::{Message, OpCode}, - rr::Record, + rr::{Name, Record}, serialize::binary::{BinDecodable, BinEncodable}, }, }; @@ -227,4 +233,20 @@ mod tests { assert_eq!(resolved, Some(entry)); } + + fn txt_record(entry: &str) -> Record { + Record::from_rdata(Name::root(), 60, RData::TXT(TXT::new(vec![entry.to_string()]))) + } + + #[test] + fn find_txt_entry_skips_unrelated_records() { + let entry = "enrtree-root:v1 e=enr-root l=link-root seq=1 sig=signature"; + let records = [ + txt_record("v=spf1 -all"), + txt_record(entry), + txt_record("enrtree-branch:YNEGZIWHOM7TOOSUATAPTM"), + ]; + + assert_eq!(find_txt_entry(&records).as_deref(), Some(entry)); + } } diff --git a/crates/net/dns/src/tree.rs b/crates/net/dns/src/tree.rs index 6e31e4e966f..c9e8f3fb6b5 100644 --- a/crates/net/dns/src/tree.rs +++ b/crates/net/dns/src/tree.rs @@ -341,6 +341,13 @@ impl fmt::Display for NodeEntry { } } +/// Returns whether the value starts with a recognized EIP-1459 entry prefix. +pub(crate) fn has_entry_prefix(value: &str) -> bool { + [ROOT_V1_PREFIX, BRANCH_PREFIX, LINK_PREFIX, ENR_PREFIX] + .iter() + .any(|prefix| value.starts_with(prefix)) +} + /// Parses the value of the key value pair fn parse_value(input: &mut &str, key: &str, err: &'static str, f: F) -> ParseEntryResult where From d229be69356313e5988e98db4183da194076a80c Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Fri, 7 Aug 2026 10:58:14 +0300 Subject: [PATCH 109/148] refactor: reuse evm for block-level tracing (#26614) Co-authored-by: Matthias Seitz Co-authored-by: Paul Lange --- crates/rpc/rpc-eth-api/src/helpers/call.rs | 69 +++----- crates/rpc/rpc-eth-api/src/helpers/trace.rs | 178 +++++++++----------- crates/rpc/rpc/src/debug.rs | 110 +++++------- crates/rpc/rpc/src/trace.rs | 10 +- 4 files changed, 152 insertions(+), 215 deletions(-) diff --git a/crates/rpc/rpc-eth-api/src/helpers/call.rs b/crates/rpc/rpc-eth-api/src/helpers/call.rs index 70beba2634e..060d2365298 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/call.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/call.rs @@ -22,7 +22,7 @@ use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks}; use reth_errors::{ProviderError, RethError}; use reth_evm::{ block::BlockExecutor, env::BlockEnvironment, execute::BlockBuilder, ConfigureEvm, Evm, - EvmEnvFor, HaltReasonFor, InspectorFor, TransactionEnvMut, TxEnvFor, + EvmEnvFor, EvmFor, HaltReasonFor, InspectorFor, TransactionEnvMut, TxEnvFor, }; use reth_node_api::BlockBody; use reth_primitives_traits::Recovered; @@ -38,7 +38,7 @@ use reth_rpc_eth_types::{ simulate::{self, EthSimulateError}, EthApiError, StateCacheDb, }; -use reth_storage_api::{BlockIdReader, ProviderTx, StateProviderBox}; +use reth_storage_api::{BlockIdReader, ProviderTx}; use revm::{ context::Block, context_interface::{result::ResultAndState, Cfg, Transaction}, @@ -299,7 +299,10 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA bundles: Vec::Network>>>, state_context: Option, mut state_override: Option, - ) -> impl Future>, Self::Error>> + Send { + ) -> impl Future>, Self::Error>> + Send + where + Self: Trace, + { async move { // Check if the vector of bundles is empty if bundles.is_empty() { @@ -354,14 +357,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA let mut all_results = Vec::with_capacity(bundles.len()); if replay_block_txs { - let mut executor = RpcNodeCore::evm_config(&this) - .executor_for_block(&mut db, block.sealed_block()) - .map_err(RethError::other) - .map_err(Self::Error::from_eth_err)?; - executor.apply_pre_execution_changes().map_err(Self::Error::from_eth_err)?; - for tx in block.transactions_recovered().take(num_txs) { - executor.execute_transaction(tx).map_err(Self::Error::from_eth_err)?; - } + this.replay_block_until(&mut db, &block, num_txs)?; } // transact all bundles @@ -533,22 +529,6 @@ pub trait Call: alloy_evm::call::caller_gas_allowance(&mut db, tx_env).map_err(Self::Error::from_eth_err) } - /// Executes the closure with the state that corresponds to the given [`BlockId`]. - fn with_state_at_block( - &self, - at: BlockId, - f: F, - ) -> impl Future> + Send - where - R: Send + 'static, - F: FnOnce(Self, StateProviderBox) -> Result + Send + 'static, - { - self.spawn_blocking_io_fut(async move |this| { - let state = this.state_at_block_id(at).await?; - f(this, state) - }) - } - /// Executes the `TxEnv` against the given [Database] without committing state /// changes. fn transact( @@ -748,37 +728,36 @@ pub trait Call: } } - /// Replays all the transactions until the target transaction is found. + /// Replays all transactions before the target transaction index on the given EVM. /// - /// All transactions before the target transaction are executed and their changes are written to - /// the _runtime_ db ([`State`]). + /// This executes on a caller provided EVM, so the target transaction can then be run on the + /// same EVM, keeping any block-scoped EVM state intact. The EVM's inspector configuration is + /// left untouched; see + /// [`Trace::inspect_transaction_in_block`] to replay without inspection and trace the target. /// - /// Note: This assumes the target transaction is in the given iterator. - /// Returns the index of the target transaction in the given iterator. - fn replay_transactions_until<'a, DB, I>( + /// If the target index is greater than or equal to the iterator length, all transactions are + /// replayed. + fn replay_transactions_until_with_evm<'a, DB, I, Txs>( &self, - db: &mut DB, - evm_env: EvmEnvFor, - transactions: I, - target_tx_hash: B256, - ) -> Result + evm: &mut EvmFor, + transactions: Txs, + target_tx_index: usize, + ) -> Result<(), Self::Error> where DB: Database> + DatabaseCommit + core::fmt::Debug, - I: IntoIterator>>, + I: InspectorFor, + Txs: IntoIterator>>, { - let mut evm = self.evm_config().evm_with_env(db, evm_env); - let mut index = 0; - for tx in transactions { - if *tx.tx_hash() == target_tx_hash { + for (index, tx) in transactions.into_iter().enumerate() { + if index == target_tx_index { // reached the target transaction break } let tx_env = self.evm_config().tx_env(tx); evm.transact_commit(tx_env).map_err(Self::Error::from_evm_err)?; - index += 1; } - Ok(index) + Ok(()) } /// diff --git a/crates/rpc/rpc-eth-api/src/helpers/trace.rs b/crates/rpc/rpc-eth-api/src/helpers/trace.rs index 9aff20b7f57..a2662147624 100644 --- a/crates/rpc/rpc-eth-api/src/helpers/trace.rs +++ b/crates/rpc/rpc-eth-api/src/helpers/trace.rs @@ -6,16 +6,12 @@ use alloy_consensus::{transaction::TxHashRef, BlockHeader}; use alloy_primitives::B256; use alloy_rpc_types_eth::{BlockId, TransactionInfo}; use futures::Future; -use reth_errors::{ProviderError, RethError}; +use reth_errors::RethError; use reth_evm::{ - block::BlockExecutor, evm::EvmFactoryExt, tracing::TracingCtx, ConfigureEvm, Database, Evm, - EvmEnvFor, EvmFor, HaltReasonFor, InspectorFor, TxEnvFor, -}; -use reth_primitives_traits::{BlockBody, Recovered, RecoveredBlock}; -use reth_revm::{ - database::StateProviderDatabase, - db::{bal::EvmDatabaseError, State}, + block::BlockExecutor, evm::EvmFactoryExt, tracing::TracingCtx, ConfigureEvm, Evm, EvmEnvFor, + EvmFor, HaltReasonFor, InspectorFor, IntoTxEnv, TxEnvFor, }; +use reth_primitives_traits::{BlockBody, BlockTy, Recovered, RecoveredBlock}; use reth_rpc_eth_types::cache::db::StateCacheDb; use reth_storage_api::{ProviderBlock, ProviderTx}; use revm::{context::Block, context_interface::result::ResultAndState}; @@ -24,85 +20,19 @@ use std::sync::Arc; /// Executes CPU heavy tasks. pub trait Trace: LoadState> + Call { - /// Executes the [`TxEnvFor`] with [`reth_evm::EvmEnv`] against the given [Database] without - /// committing state changes. - fn inspect( + /// Executes the [`TxEnvFor`] with [`reth_evm::EvmEnv`] against the given [`StateCacheDb`] + /// without committing state changes. + fn inspect<'a>( &self, - db: DB, + db: &'a mut StateCacheDb, evm_env: EvmEnvFor, - tx_env: TxEnvFor, - inspector: I, - ) -> Result>, Self::Error> - where - DB: Database>, - I: InspectorFor, - { - let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env, inspector); - evm.transact(tx_env).map_err(Self::Error::from_evm_err) - } - - /// Executes the transaction on top of the given [`BlockId`] with a tracer configured by the - /// config. - /// - /// The callback is then called with the [`TracingInspector`] and the [`ResultAndState`] after - /// the configured [`reth_evm::EvmEnv`] was inspected. - /// - /// Caution: this is blocking - fn trace_at( - &self, - evm_env: EvmEnvFor, - tx_env: TxEnvFor, - config: TracingInspectorConfig, - at: BlockId, - f: F, - ) -> impl Future> + Send - where - R: Send + 'static, - F: FnOnce( - TracingInspector, - ResultAndState>, - ) -> Result - + Send - + 'static, - { - self.with_state_at_block(at, move |this, state| { - let mut db = State::builder().with_database(StateProviderDatabase::new(state)).build(); - let mut inspector = TracingInspector::new(config); - let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?; - f(inspector, res) - }) - } - - /// Same as [`trace_at`](Self::trace_at) but also provides the used database to the callback. - /// - /// Executes the transaction on top of the given [`BlockId`] with a tracer configured by the - /// config. - /// - /// The callback is then called with the [`TracingInspector`] and the [`ResultAndState`] after - /// the configured [`reth_evm::EvmEnv`] was inspected. - fn spawn_trace_at_with_state( - &self, - evm_env: EvmEnvFor, - tx_env: TxEnvFor, - config: TracingInspectorConfig, - at: BlockId, - f: F, - ) -> impl Future> + Send - where - F: FnOnce( - TracingInspector, - ResultAndState>, - StateCacheDb, - ) -> Result - + Send - + 'static, - R: Send + 'static, - { - self.spawn_with_state_at_block(at, move |this, mut db| { - let mut inspector = TracingInspector::new(config); - let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?; - f(inspector, res, db) - }) + tx_env: impl IntoTxEnv>, + inspector: impl InspectorFor, + ) -> Result>, Self::Error> { + self.evm_config() + .evm_with_env_and_inspector(db, evm_env, inspector) + .transact(tx_env) + .map_err(Self::Error::from_evm_err) } /// Retrieves the transaction if it exists and returns its trace. @@ -170,22 +100,21 @@ pub trait Trace: LoadState> + Call { }; let (tx, tx_info) = transaction.split(); - let evm_env = self.evm_env_for_header(block.sealed_block().sealed_header())?; - // we need to get the state of the parent block because we're essentially replaying the // block the transaction is included in let parent_block = block.parent_hash(); self.spawn_with_state_at_block(parent_block, move |this, mut db| { - let block_txs = block.transactions_recovered(); - - this.apply_pre_execution_changes(&block, &mut db)?; - - // replay all transactions prior to the targeted transaction - this.replay_transactions_until(&mut db, evm_env.clone(), block_txs, *tx.tx_hash())?; - - let tx_env = this.evm_config().tx_env(tx); - let res = this.inspect(&mut db, evm_env, tx_env, &mut inspector)?; + let (res, _) = this.inspect_transaction_in_block( + &block, + &mut db, + &mut inspector, + // index should always be available because `transaction_and_block` only + // returns transactions included in a block + tx_info.index.expect("transaction_and_block only returns block transactions") + as usize, + tx, + )?; f(tx_info, inspector, res, db) }) .await @@ -193,6 +122,59 @@ pub trait Trace: LoadState> + Call { } } + /// Replays all transactions before the target transaction index. + /// + /// All transactions before the target transaction are executed and their changes are written to + /// the _runtime_ db ([`StateCacheDb`]). + /// + /// If the target index is greater than or equal to the block's transaction count, all + /// transactions are replayed. + fn replay_block_until( + &self, + db: &mut StateCacheDb, + block: &RecoveredBlock>, + target_tx_index: usize, + ) -> Result<(), Self::Error> { + self.apply_pre_execution_changes(block, db)?; + + let evm_env = self.evm_env_for_header(block.sealed_block().sealed_header())?; + let mut evm = self.evm_config().evm_with_env(db, evm_env); + self.replay_transactions_until_with_evm( + &mut evm, + block.transactions_recovered(), + target_tx_index, + ) + } + + /// Replays all transactions before the target transaction without inspection, then executes + /// the target transaction with the configured inspector, all on the given EVM. + #[expect(clippy::type_complexity)] + fn inspect_transaction_in_block<'a>( + &self, + block: &RecoveredBlock>, + db: &'a mut StateCacheDb, + inspector: impl InspectorFor, + target_tx_index: usize, + target_tx_env: impl IntoTxEnv>, + ) -> Result<(ResultAndState>, EvmEnvFor), Self::Error> { + let block_txs = block.transactions_recovered(); + + self.apply_pre_execution_changes(block, db)?; + + let evm_env = self.evm_env_for_header(block.sealed_block().sealed_header())?; + let mut evm = self.evm_config().evm_with_env_and_inspector(db, evm_env, inspector); + + evm.disable_inspector(); + self.replay_transactions_until_with_evm(&mut evm, block_txs, target_tx_index)?; + evm.enable_inspector(); + + let res = evm.transact(target_tx_env).map_err(Self::Error::from_evm_err)?; + + let (_, evm_env) = evm.finish(); + + Ok((res, evm_env)) + } + /// Executes all transactions of a block up to a given index. /// /// If a `highest_index` is given, this will only execute the first `highest_index` @@ -333,8 +315,8 @@ pub trait Trace: LoadState> + Call { /// 2. configures the EVM env /// 3. loops over all transactions and executes them /// 4. calls the callback with the transaction info, the execution result, the changed state - /// _after_ the transaction [`StateProviderDatabase`] and the database that points to the - /// state right _before_ the transaction. + /// _after_ the transaction [`StateCacheDb`] and the database that points to the state right + /// _before_ the transaction. fn trace_block_with( &self, block_id: BlockId, diff --git a/crates/rpc/rpc/src/debug.rs b/crates/rpc/rpc/src/debug.rs index 24bc871691d..9860eae2fc7 100644 --- a/crates/rpc/rpc/src/debug.rs +++ b/crates/rpc/rpc/src/debug.rs @@ -124,33 +124,33 @@ where eth_api.apply_pre_execution_changes(&block, &mut db)?; + let block_env = evm_env.block_env.clone(); + let mut transactions = block.transactions_recovered().enumerate().peekable(); - let mut inspector = DebugInspector::new(opts).map_err(Eth::Error::from_eth_err)?; + let inspector = DebugInspector::new(opts).map_err(Eth::Error::from_eth_err)?; + let mut evm = + eth_api.evm_config().evm_with_env_and_inspector(&mut db, evm_env, inspector); while let Some((index, tx)) = transactions.next() { - let tx_hash = *tx.tx_hash(); let tx_env = eth_api.evm_config().tx_env(tx); - let res = eth_api.inspect( - &mut db, - evm_env.clone(), - tx_env.clone(), - &mut inspector, - )?; + let res = evm.transact(tx_env.clone()).map_err(Eth::Error::from_evm_err)?; + + let (db, inspector, _) = evm.components_mut(); let result = inspector .get_result( Some(TransactionContext { block_hash: Some(block.hash()), - tx_hash: Some(tx_hash), + tx_hash: Some(*tx.tx_hash()), tx_index: Some(index), }), &tx_env, - &evm_env.block_env, + &block_env, &res, - &mut db, + db, ) .map_err(Eth::Error::from_eth_err)?; - results.push(TraceResult::Success { result, tx_hash: Some(tx_hash) }); + results.push(TraceResult::Success { result, tx_hash: Some(*tx.tx_hash()) }); if transactions.peek().is_some() { inspector.fuse().map_err(Eth::Error::from_eth_err)?; // need to apply the state changes of this transaction before executing the @@ -229,39 +229,32 @@ where None => return Err(EthApiError::TracingTransactionNotFound.into()), Some(res) => res, }; - let evm_env = self.eth_api().evm_env_for_header(block.sealed_block().sealed_header())?; - - // we need to get the state of the parent block because we're essentially replaying the - // block the transaction is included in - let state_at: BlockId = block.parent_hash().into(); - let block_hash = block.hash(); self.eth_api() - .spawn_with_state_at_block(state_at, move |eth_api, mut db| { - let block_txs = block.transactions_recovered(); - + .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { // configure env for the target transaction - let tx = transaction.into_recovered(); + let (tx, tx_info) = transaction.split(); - eth_api.apply_pre_execution_changes(&block, &mut db)?; + // index should always be available because `transaction_and_block` only + // returns transactions included in a block + let index = + tx_info.index.expect("transaction_and_block only returns block transactions") + as usize; - // replay all transactions prior to the targeted transaction - let index = eth_api.replay_transactions_until( + let mut inspector = DebugInspector::new(opts).map_err(Eth::Error::from_eth_err)?; + let tx_env = eth_api.evm_config().tx_env(&tx); + let (res, evm_env) = eth_api.inspect_transaction_in_block( + &block, &mut db, - evm_env.clone(), - block_txs, - *tx.tx_hash(), + &mut inspector, + index, + tx_env.clone(), )?; - let tx_env = eth_api.evm_config().tx_env(&tx); - - let mut inspector = DebugInspector::new(opts).map_err(Eth::Error::from_eth_err)?; - let res = - eth_api.inspect(&mut db, evm_env.clone(), tx_env.clone(), &mut inspector)?; let trace = inspector .get_result( Some(TransactionContext { - block_hash: Some(block_hash), + block_hash: Some(block.hash()), tx_index: Some(index), tx_hash: Some(*tx.tx_hash()), }), @@ -357,23 +350,12 @@ where let evm_env = self.eth_api().evm_env_for_header(block.sealed_block().sealed_header())?; - // execute after the parent block, replaying `tx_index` transactions - let state_at = block.parent_hash(); - self.eth_api() - .spawn_with_state_at_block(state_at, move |eth_api, mut db| { - // 1. apply pre-execution changes - eth_api.apply_pre_execution_changes(&block, &mut db)?; - - // 2. replay the required number of transactions - eth_api.replay_transactions_until( - &mut db, - evm_env.clone(), - block.transactions_recovered(), - *block.body().transactions()[tx_index].tx_hash(), - )?; + .spawn_with_state_at_block(block.parent_hash(), move |eth_api, mut db| { + // 1. replay the required number of transactions + eth_api.replay_block_until(&mut db, &block, tx_index)?; - // 3. now execute the trace call on this state + // 2. now execute the trace call on this state let (evm_env, tx_env) = eth_api.prepare_call_env(evm_env, call, &mut db, overrides)?; @@ -442,16 +424,8 @@ where if replay_block_txs { // only need to replay the transactions in the block if not all transactions are // to be replayed - eth_api.apply_pre_execution_changes(&block, &mut db)?; - - let transactions = block.transactions_recovered().take(num_txs); - // Execute all transactions until index - for tx in transactions { - let tx_env = eth_api.evm_config().tx_env(tx); - let res = eth_api.transact(&mut db, evm_env.clone(), tx_env)?; - db.commit(res.state); - } + eth_api.replay_block_until(&mut db, &block, num_txs)?; } // Trace all bundles @@ -740,21 +714,23 @@ where eth_api.apply_pre_execution_changes(&block, &mut db)?; let mut roots = Vec::with_capacity(block.body().transactions().len()); + let mut evm = eth_api.evm_config().evm_with_env(&mut db, evm_env); for tx in block.transactions_recovered() { let tx_env = eth_api.evm_config().tx_env(tx); - { - let mut evm = eth_api.evm_config().evm_with_env(&mut db, evm_env.clone()); - evm.transact_commit(tx_env).map_err(Eth::Error::from_evm_err)?; - } + evm.transact_commit(tx_env).map_err(Eth::Error::from_evm_err)?; + + let state = evm.db_mut(); // Merge transitions into cumulative bundle_state - db.merge_transitions(BundleRetention::PlainState); + state.merge_transitions(BundleRetention::PlainState); // Compute state root from the accumulated state changes - let hashed_state = db + let hashed_state = state + .database + .hashed_post_state(&state.bundle_state) + .map_err(Eth::Error::from_eth_err)?; + let root = state .database - .hashed_post_state(&db.bundle_state) + .state_root(hashed_state) .map_err(Eth::Error::from_eth_err)?; - let root = - db.database.state_root(hashed_state).map_err(Eth::Error::from_eth_err)?; roots.push(root); } diff --git a/crates/rpc/rpc/src/trace.rs b/crates/rpc/rpc/src/trace.rs index bc8b88a350b..43b1158890a 100644 --- a/crates/rpc/rpc/src/trace.rs +++ b/crates/rpc/rpc/src/trace.rs @@ -19,7 +19,6 @@ use async_trait::async_trait; use futures::StreamExt; use jsonrpsee::core::RpcResult; use reth_chainspec::{ChainSpecProvider, EthereumHardforks}; -use reth_evm::ConfigureEvm; use reth_primitives_traits::{BlockBody, BlockHeader}; use reth_rpc_api::TraceApiServer; use reth_rpc_convert::RpcTxReq; @@ -128,12 +127,13 @@ where .map(::Transaction::pooled_into_consensus); let (evm_env, at) = self.eth_api().evm_env_at(block_id.unwrap_or_default()).await?; - let tx_env = self.eth_api().evm_config().tx_env(tx); - - let config = TracingInspectorConfig::from_parity_config(&trace_types); self.eth_api() - .spawn_trace_at_with_state(evm_env, tx_env, config, at, move |inspector, res, db| { + .spawn_with_state_at_block(at, move |this, mut db| { + let mut inspector = + TracingInspector::new(TracingInspectorConfig::from_parity_config(&trace_types)); + let res = this.inspect(&mut db, evm_env, tx, &mut inspector)?; + inspector .into_parity_builder() .into_trace_results_with_state(&res, &trace_types, &db) From 6cda1bfa0a4b366a9dbe82ca2b9621da52502b4d Mon Sep 17 00:00:00 2001 From: Faisal Firdani <50091513+zexoverz@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:02:37 +0700 Subject: [PATCH 110/148] fix(cli): download enough history when the final snapshot chunk is partial (#26607) Co-authored-by: Matthias Seitz --- crates/cli/commands/src/download/manifest.rs | 120 +++++++++++++++---- 1 file changed, 95 insertions(+), 25 deletions(-) diff --git a/crates/cli/commands/src/download/manifest.rs b/crates/cli/commands/src/download/manifest.rs index cc3f3e1adf9..0030ba554f7 100644 --- a/crates/cli/commands/src/download/manifest.rs +++ b/crates/cli/commands/src/download/manifest.rs @@ -318,12 +318,8 @@ impl SnapshotManifest { // Calculate which chunks to include let start_chunk = match distance { - Some(dist) => { - // We need chunks covering the last `dist` blocks - let needed_blocks = dist.min(chunked.total_blocks); - let needed_chunks = needed_blocks.div_ceil(chunked.blocks_per_file); - num_chunks.saturating_sub(needed_chunks) - } + // Include the tail chunks that cover at least `dist` blocks. + Some(dist) => num_chunks.saturating_sub(chunked.tail_chunks_for_distance(dist)), None => 0, // All chunks }; @@ -363,11 +359,7 @@ impl SnapshotManifest { let num_chunks = chunked.num_chunks(); let start_chunk = match distance { - Some(dist) => { - let needed_blocks = dist.min(chunked.total_blocks); - let needed_chunks = needed_blocks.div_ceil(chunked.blocks_per_file); - num_chunks.saturating_sub(needed_chunks) - } + Some(dist) => num_chunks.saturating_sub(chunked.tail_chunks_for_distance(dist)), None => 0, }; @@ -409,11 +401,7 @@ impl SnapshotManifest { } let num_chunks = chunked.chunk_sizes.len() as u64; let start_chunk = match distance { - Some(dist) => { - let needed = dist.min(chunked.total_blocks); - let needed_chunks = needed.div_ceil(chunked.blocks_per_file); - num_chunks.saturating_sub(needed_chunks) - } + Some(dist) => num_chunks.saturating_sub(chunked.tail_chunks_for_distance(dist)), None => 0, }; chunked.chunk_sizes[start_chunk as usize..].iter().sum() @@ -436,11 +424,7 @@ impl SnapshotManifest { ComponentManifest::Chunked(chunked) => { let num_chunks = chunked.num_chunks(); let start_chunk = match distance { - Some(dist) => { - let needed = dist.min(chunked.total_blocks); - let needed_chunks = needed.div_ceil(chunked.blocks_per_file); - num_chunks.saturating_sub(needed_chunks) - } + Some(dist) => num_chunks.saturating_sub(chunked.tail_chunks_for_distance(dist)), None => 0, }; @@ -457,10 +441,7 @@ impl SnapshotManifest { return if self.component(ty).is_some() { 1 } else { 0 }; }; match distance { - Some(dist) => { - let needed = dist.min(chunked.total_blocks); - needed.div_ceil(chunked.blocks_per_file) - } + Some(dist) => chunked.tail_chunks_for_distance(dist), None => chunked.num_chunks(), } } @@ -490,6 +471,19 @@ impl ChunkedArchive { self.total_blocks.div_ceil(self.blocks_per_file) } + /// Returns the number of tail chunks required to cover at least `distance` blocks from the + /// tip. + pub fn tail_chunks_for_distance(&self, distance: u64) -> u64 { + let needed = distance.min(self.total_blocks); + if needed == 0 { + return 0; + } + + // The first needed block determines the earliest chunk, including for a partial tail. + let first_chunk = (self.total_blocks - needed) / self.blocks_per_file; + self.num_chunks() - first_chunk + } + /// Returns the extracted plain-output size for one chunk. pub fn chunk_output_size(&self, index: usize) -> u64 { self.chunk_decompressed_sizes.get(index).copied().unwrap_or_else(|| { @@ -1034,6 +1028,82 @@ mod tests { assert_eq!(m.chunks_for_distance(SnapshotComponentType::State, Some(100)), 1); } + /// A snapshot whose final chunk is partial: `total_blocks` is not a multiple of + /// `blocks_per_file`, so the last chunk holds fewer than `blocks_per_file` blocks. + /// Three chunks: `[0, 500k)`, `[500k, 1M)`, and `[1M, 1.005M)` — the last holds 5_000 blocks. + fn partial_tail_manifest() -> SnapshotManifest { + let mut components = BTreeMap::new(); + components.insert( + "transactions".to_string(), + ComponentManifest::Chunked(ChunkedArchive { + blocks_per_file: 500_000, + total_blocks: 1_005_000, + chunk_sizes: vec![10, 20, 30], + chunk_decompressed_sizes: vec![100, 200, 300], + chunk_output_files: vec![vec![], vec![], vec![]], + }), + ); + SnapshotManifest { + block: 1_005_000, + chain_id: 1, + storage_version: 2, + timestamp: 0, + base_url: Some("https://example.com".to_string()), + reth_version: None, + components, + } + } + + #[test] + fn tail_chunks_for_distance_accounts_for_partial_final_chunk() { + let chunked = ChunkedArchive { + blocks_per_file: 500_000, + total_blocks: 1_005_000, + chunk_sizes: vec![10, 20, 30], + chunk_decompressed_sizes: vec![], + chunk_output_files: vec![], + }; + // The final chunk holds only 5_000 blocks, so covering 10_064 blocks needs the last two + // chunks. The naive `distance.div_ceil(blocks_per_file)` returns 1 here and falls short. + assert_eq!(chunked.tail_chunks_for_distance(10_064), 2); + // A distance that fits inside the final chunk still needs just one. + assert_eq!(chunked.tail_chunks_for_distance(5_000), 1); + // One block more than the final chunk holds must pull in the previous chunk. + assert_eq!(chunked.tail_chunks_for_distance(5_001), 2); + assert_eq!(chunked.tail_chunks_for_distance(1), 1); + // Covering everything selects all chunks and never exceeds the chunk count. + assert_eq!(chunked.tail_chunks_for_distance(1_005_000), 3); + assert_eq!(chunked.tail_chunks_for_distance(u64::MAX), 3); + } + + #[test] + fn partial_final_chunk_downloads_enough_history() { + let m = partial_tail_manifest(); + // The published snapshot's final chunk is partial (5_000 blocks). A --full download + // asking for 10_064 blocks must pull the last two chunks, then prune the surplus, + // rather than the single short tail chunk the old distance math selected. + assert_eq!(m.chunks_for_distance(SnapshotComponentType::Transactions, Some(10_064)), 2); + + let urls = m.archive_urls_for_distance(SnapshotComponentType::Transactions, Some(10_064)); + assert_eq!(urls.len(), 2); + assert_eq!(urls[0], "https://example.com/transactions-500000-999999.tar.zst"); + assert_eq!(urls[1], "https://example.com/transactions-1000000-1499999.tar.zst"); + + let archives = + m.snapshot_archives_for_distance(SnapshotComponentType::Transactions, Some(10_064)); + assert_eq!(archives.len(), 2); + + // Reported sizes cover the selected tail chunks (indices 1 and 2). + assert_eq!(m.size_for_distance(SnapshotComponentType::Transactions, Some(10_064)), 20 + 30); + assert_eq!( + m.output_size_for_distance(SnapshotComponentType::Transactions, Some(10_064)), + 200 + 300 + ); + + // A distance that fits inside the final chunk still selects a single chunk. + assert_eq!(m.chunks_for_distance(SnapshotComponentType::Transactions, Some(5_000)), 1); + } + #[test] fn chunks_for_distance_missing() { let m = test_manifest(); From 9dd31ea6a8b3aff565fa6064ef3e93b13a5b40d3 Mon Sep 17 00:00:00 2001 From: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com> Date: Fri, 7 Aug 2026 15:39:41 +0530 Subject: [PATCH 111/148] fix(net): preserve eth72 announcement cell masks (#26573) --- crates/net/eth-wire-types/src/broadcast.rs | 49 +++++++++++++++++++--- 1 file changed, 44 insertions(+), 5 deletions(-) diff --git a/crates/net/eth-wire-types/src/broadcast.rs b/crates/net/eth-wire-types/src/broadcast.rs index ff0af6cd4f0..64596203e8f 100644 --- a/crates/net/eth-wire-types/src/broadcast.rs +++ b/crates/net/eth-wire-types/src/broadcast.rs @@ -1034,7 +1034,7 @@ impl DedupPayload for NewPooledTransactionHashes72 { } fn dedup(self) -> PartiallyValidData { - let Self { hashes, mut sizes, mut types, .. } = self; + let Self { hashes, mut sizes, mut types, cell_mask } = self; let mut deduped_data = B256Map::with_capacity_and_hasher(hashes.len(), Default::default()); @@ -1044,7 +1044,7 @@ impl DedupPayload for NewPooledTransactionHashes72 { } } - PartiallyValidData::from_raw_data_eth72(deduped_data) + PartiallyValidData::from_raw_data_eth72_with_cell_mask(deduped_data, cell_mask) } } @@ -1161,6 +1161,8 @@ pub struct PartiallyValidData { #[into_iterator] data: B256Map, version: Option, + /// The eth/72 message-level cell mask, if present. + cell_mask: Option, } handle_mempool_data_map_impl!(PartiallyValidData, ); @@ -1168,7 +1170,7 @@ handle_mempool_data_map_impl!(PartiallyValidData, ); impl PartiallyValidData { /// Wraps raw data. pub const fn from_raw_data(data: B256Map, version: Option) -> Self { - Self { data, version } + Self { data, version, cell_mask: None } } /// Wraps raw data with version [`EthVersion::Eth72`]. @@ -1176,6 +1178,14 @@ impl PartiallyValidData { Self::from_raw_data(data, Some(EthVersion::Eth72)) } + /// Wraps raw data with an eth/72 message-level cell mask. + pub const fn from_raw_data_eth72_with_cell_mask( + data: B256Map, + cell_mask: Option, + ) -> Self { + Self { data, version: Some(EthVersion::Eth72), cell_mask } + } + /// Wraps raw data with version [`EthVersion::Eth68`]. pub const fn from_raw_data_eth68(data: B256Map) -> Self { Self::from_raw_data(data, Some(EthVersion::Eth68)) @@ -1210,6 +1220,11 @@ impl PartiallyValidData { self.version } + /// Returns the eth/72 message-level cell mask, if present. + pub const fn eth72_cell_mask(&self) -> Option { + self.cell_mask + } + /// Destructs returning the validated data. pub fn into_data(self) -> B256Map { self.data @@ -1225,6 +1240,8 @@ pub struct ValidAnnouncementData { #[into_iterator] data: B256Map, version: EthVersion, + /// The eth/72 message-level cell mask, if present. + cell_mask: Option, } handle_mempool_data_map_impl!(ValidAnnouncementData,); @@ -1242,11 +1259,16 @@ impl ValidAnnouncementData { /// from an announcement, should have some [`EthVersion`]. Panics if [`PartiallyValidData`] has /// version set to `None`. pub fn from_partially_valid_data(data: PartiallyValidData) -> Self { - let PartiallyValidData { data, version } = data; + let PartiallyValidData { data, version, cell_mask } = data; let version = version.expect("should have eth version for conversion"); - Self { data, version } + Self { data, version, cell_mask } + } + + /// Returns the eth/72 message-level cell mask, if present. + pub const fn eth72_cell_mask(&self) -> Option { + self.cell_mask } /// Destructs returning the validated data. @@ -1670,6 +1692,23 @@ mod tests { assert!(matches!(result, Err(alloy_rlp::Error::InputTooShort))); } + #[test] + fn eth_72_dedup_preserves_message_cell_mask() { + let cell_mask = Some(B128::repeat_byte(0x11)); + let announcement = NewPooledTransactionHashes72 { + types: vec![3], + sizes: vec![128], + hashes: vec![B256::from([1u8; 32])], + cell_mask, + }; + + let partially_valid = announcement.dedup(); + assert_eq!(partially_valid.eth72_cell_mask(), cell_mask); + + let valid = ValidAnnouncementData::from_partially_valid_data(partially_valid); + assert_eq!(valid.eth72_cell_mask(), cell_mask); + } + #[test] fn request_hashes_retain_count_keep_subset() { let mut hashes = RequestTxHashes::new( From c7d94e622e0cd240961de7fa15d71d33ac900d07 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Sat, 8 Aug 2026 19:51:43 +0200 Subject: [PATCH 112/148] fix(ci): resolve nightly Clippy failures (#26625) --- Cargo.lock | 19 +++++++++++++++---- Cargo.toml | 4 +++- crates/engine/local/src/miner.rs | 4 +--- crates/net/ecies/src/codec.rs | 4 +--- crates/stages/api/src/pipeline/mod.rs | 25 +++++++------------------ 5 files changed, 27 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 93af0aca20a..b33be037034 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1509,13 +1509,13 @@ dependencies = [ [[package]] name = "async-trait" -version = "0.1.89" +version = "0.1.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" +checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 2.0.118", + "syn 3.0.3", ] [[package]] @@ -2973,7 +2973,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7ab67060fc6b8ef687992d439ca0fa36e7ed17e9a0b16b25b601e8757df720de" dependencies = [ "data-encoding", - "syn 2.0.118", + "syn 1.0.109", ] [[package]] @@ -12226,6 +12226,17 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "syn" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn-solidity" version = "1.6.1" diff --git a/Cargo.toml b/Cargo.toml index bae4ee21f90..b23f6556a63 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -203,7 +203,7 @@ manual_clamp = "warn" manual_is_variant_and = "warn" manual_string_new = "warn" match_same_arms = "warn" -missing-const-for-fn = "warn" +missing_const_for_fn = "warn" mutex_integer = "warn" naive_bytecount = "warn" needless_bitwise_bool = "warn" @@ -218,6 +218,8 @@ read_zero_byte_vec = "warn" result_large_err = "allow" redundant_clone = "warn" redundant_else = "warn" +# TODO: Remove once https://github.com/rust-lang/rust-clippy/issues/17525 is resolved. +redundant_field_names = "allow" single_char_pattern = "warn" string_lit_as_bytes = "warn" string_lit_chars_any = "warn" diff --git a/crates/engine/local/src/miner.rs b/crates/engine/local/src/miner.rs index 920ec2632d8..c9047814a38 100644 --- a/crates/engine/local/src/miner.rs +++ b/crates/engine/local/src/miner.rs @@ -112,21 +112,19 @@ impl Future for MiningMode { return Poll::Ready(()); } } - Poll::Pending } Self::Interval(interval) => { if interval.poll_tick(cx).is_ready() { return Poll::Ready(()) } - Poll::Pending } Self::Trigger(trigger) => { if trigger.poll_next_unpin(cx).is_ready() { return Poll::Ready(()) } - Poll::Pending } } + Poll::Pending } } diff --git a/crates/net/ecies/src/codec.rs b/crates/net/ecies/src/codec.rs index e5a3289b7f2..5fb11739c26 100644 --- a/crates/net/ecies/src/codec.rs +++ b/crates/net/ecies/src/codec.rs @@ -159,18 +159,16 @@ impl Encoder for ECIESCodec { EgressECIESValue::Auth => { self.state = ECIESState::Ack; self.ecies.write_auth(buf); - Ok(()) } EgressECIESValue::Ack => { self.state = ECIESState::InitialHeader; self.ecies.write_ack(buf); - Ok(()) } EgressECIESValue::Message(data) => { self.ecies.write_header(buf, data.len()); self.ecies.write_body(buf, &data); - Ok(()) } } + Ok(()) } } diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index 142e5ad4aa2..0c2f3b668d6 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -603,15 +603,6 @@ impl Pipeline { provider_rw.commit()?; } - - // We unwind because of a validation error. If the unwind itself - // fails, we bail entirely, - // otherwise we restart the execution loop from the - // beginning. - Ok(Some(ControlFlow::Unwind { - target: prev_checkpoint.unwrap_or_default().block_number, - bad_block: block, - })) } BlockErrorKind::Execution(execution_error) => { error!( @@ -620,17 +611,15 @@ impl Pipeline { bad_block = %block.block.number, "Stage encountered an execution error: {execution_error}" ); - - // We unwind because of an execution error. If the unwind itself - // fails, we bail entirely, - // otherwise we restart - // the execution loop from the beginning. - Ok(Some(ControlFlow::Unwind { - target: prev_checkpoint.unwrap_or_default().block_number, - bad_block: block, - })) } } + + // We unwind because of a block error. If the unwind itself fails, we bail entirely; + // otherwise, we restart the execution loop from the beginning. + Ok(Some(ControlFlow::Unwind { + target: prev_checkpoint.unwrap_or_default().block_number, + bad_block: block, + })) } else if let StageError::MissingStaticFileData { block, segment } = err { error!( target: "sync::pipeline", From 6646bb334c35d37123902ada0a2eb4ba97a14b9e Mon Sep 17 00:00:00 2001 From: William Law Date: Mon, 10 Aug 2026 11:24:06 -0400 Subject: [PATCH 113/148] feat(snapshots): resolve base url for static files (#26576) Co-authored-by: Cursor Co-authored-by: Matthias Seitz --- crates/cli/commands/src/download/manifest.rs | 273 +++++++++++++++++-- crates/cli/commands/src/download/planning.rs | 58 ++++ 2 files changed, 312 insertions(+), 19 deletions(-) diff --git a/crates/cli/commands/src/download/manifest.rs b/crates/cli/commands/src/download/manifest.rs index 0030ba554f7..174ec1c59d0 100644 --- a/crates/cli/commands/src/download/manifest.rs +++ b/crates/cli/commands/src/download/manifest.rs @@ -9,6 +9,7 @@ use std::{ path::{Path, PathBuf}, }; use tracing::info; +use url::Url; fn is_zero(value: &u64) -> bool { *value == 0 @@ -20,13 +21,17 @@ fn is_zero(value: &u64) -> bool { /// segments like transactions, receipts, etc). Chunked components use `blocks_per_file` to /// define the block range per archive, matching reth's static file segment boundaries. /// -/// Archive naming convention for chunked components: -/// `{component}-{start_block}-{end_block}.tar.zst` +/// Archive paths are resolved relative to [`SnapshotManifest::base_url`] via URL joining. +/// Single archives use [`SingleArchive::file`]; chunked archives default to +/// `{component}-{start_block}-{end_block}.tar.zst`, or use [`ChunkedArchive::chunk_files`] +/// when present so publishers can place finalized and tip chunks under different prefixes: /// -/// For example with `blocks_per_file: 500000` and `total_blocks: 1500000`: -/// `transactions-0-499999.tar.zst` -/// `transactions-500000-999999.tar.zst` -/// `transactions-1000000-1499999.tar.zst` +/// ```text +/// base_url = https://example.com/mainnet +/// static_files/transactions-0-499999.tar.zst +/// 1700000/transactions-500000-999999.tar.zst +/// 1700000/state.tar.zst +/// ``` #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SnapshotManifest { /// Block number this snapshot was taken at. @@ -63,6 +68,9 @@ pub enum ComponentManifest { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct SingleArchive { /// Archive file name (relative to base_url). + /// + /// Must be a relative path (no leading `/`). Nested paths like `1700000/state.tar.zst` + /// are supported when `base_url` points at the snapshot root. pub file: String, /// Compressed archive size in bytes. pub size: u64, @@ -99,6 +107,19 @@ pub struct ChunkedArchive { /// `chunk_output_files`. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub chunk_decompressed_sizes: Vec, + /// Archive path for each chunk, relative to [`SnapshotManifest::base_url`], ordered from + /// first to last. + /// + /// When empty (older manifests), downloaders fall back to the default + /// `{component}-{start}-{end}.tar.zst` name. When set, the length must equal the chunk + /// count and each entry is joined with `base_url` so publishers can place chunks under + /// different prefixes (for example `static_files/…` for finalized chunks and + /// `{timestamp}/…` for the tip chunk). + /// + /// Paths must be relative (no leading `/`). A leading slash is treated as host-absolute by + /// URL joining and would drop the `base_url` path prefix. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub chunk_files: Vec, /// Expected extracted plain files per chunk, ordered from first to last. /// /// This is the authoritative integrity source for the modular download path. @@ -281,16 +302,17 @@ impl SnapshotManifest { match component { ComponentManifest::Single(single) => { - vec![format!("{}/{}", self.base_url_or_empty(), single.file)] + vec![resolve_archive_url(self.base_url_or_empty(), &single.file)] } ComponentManifest::Chunked(chunked) => { let key = ty.key(); let num_chunks = chunked.num_chunks(); (0..num_chunks) .map(|i| { - let start = i * chunked.blocks_per_file; - let end = (i + 1) * chunked.blocks_per_file - 1; - format!("{}/{key}-{start}-{end}.tar.zst", self.base_url_or_empty()) + resolve_archive_url( + self.base_url_or_empty(), + &chunked.chunk_relative_path(key, i), + ) }) .collect() } @@ -310,7 +332,7 @@ impl SnapshotManifest { match component { ComponentManifest::Single(single) => { - vec![format!("{}/{}", self.base_url_or_empty(), single.file)] + vec![resolve_archive_url(self.base_url_or_empty(), &single.file)] } ComponentManifest::Chunked(chunked) => { let key = ty.key(); @@ -325,9 +347,10 @@ impl SnapshotManifest { (start_chunk..num_chunks) .map(|i| { - let start = i * chunked.blocks_per_file; - let end = (i + 1) * chunked.blocks_per_file - 1; - format!("{}/{key}-{start}-{end}.tar.zst", self.base_url_or_empty()) + resolve_archive_url( + self.base_url_or_empty(), + &chunked.chunk_relative_path(key, i), + ) }) .collect() } @@ -347,7 +370,7 @@ impl SnapshotManifest { match component { ComponentManifest::Single(single) => { vec![SnapshotArchive { - url: format!("{}/{}", self.base_url_or_empty(), single.file), + url: resolve_archive_url(self.base_url_or_empty(), &single.file), file_name: single.file.clone(), size: single.size, blake3: single.blake3.clone(), @@ -365,15 +388,13 @@ impl SnapshotManifest { (start_chunk..num_chunks) .map(|i| { - let start = i * chunked.blocks_per_file; - let end = (i + 1) * chunked.blocks_per_file - 1; - let file_name = format!("{key}-{start}-{end}.tar.zst"); + let file_name = chunked.chunk_relative_path(key, i); let size = chunked.chunk_sizes.get(i as usize).copied().unwrap_or_default(); let output_files = chunked.chunk_output_files.get(i as usize).cloned().unwrap_or_default(); SnapshotArchive { - url: format!("{}/{}", self.base_url_or_empty(), file_name), + url: resolve_archive_url(self.base_url_or_empty(), &file_name), file_name, size, blake3: None, @@ -484,6 +505,26 @@ impl ChunkedArchive { self.num_chunks() - first_chunk } + /// Returns `true` when `chunk_files` is empty or has exactly one path per chunk. + pub fn chunk_files_are_consistent(&self) -> bool { + self.chunk_files.is_empty() || self.chunk_files.len() as u64 == self.num_chunks() + } + + /// Returns the archive path for chunk `index`, relative to the manifest base URL. + /// + /// Uses [`Self::chunk_files`] when it has exactly one entry per chunk; otherwise the + /// default `{key}-{start}-{end}.tar.zst` name. + pub fn chunk_relative_path(&self, key: &str, index: u64) -> String { + if self.chunk_files.len() as u64 == self.num_chunks() && + let Some(path) = self.chunk_files.get(index as usize) + { + return path.clone(); + } + let start = index * self.blocks_per_file; + let end = (index + 1) * self.blocks_per_file - 1; + format!("{key}-{start}-{end}.tar.zst") + } + /// Returns the extracted plain-output size for one chunk. pub fn chunk_output_size(&self, index: usize) -> u64 { self.chunk_decompressed_sizes.get(index).copied().unwrap_or_else(|| { @@ -507,6 +548,36 @@ impl ChunkedArchive { } } +/// Joins an archive path relative to `base_url`. +/// +/// Ensures directory semantics for `base_url` so the last path segment is not replaced when +/// joining nested paths like `static_files/headers-0-499999.tar.zst`. Leading slashes on +/// `relative_path` are stripped so publisher paths stay relative to the base prefix. +fn resolve_archive_url(base_url: &str, relative_path: &str) -> String { + let relative_path = relative_path.trim_start_matches('/'); + + if base_url.is_empty() { + return relative_path.to_string(); + } + + let Ok(mut base) = Url::parse(base_url) else { + return format!("{}/{}", base_url.trim_end_matches('/'), relative_path); + }; + + // Url::join replaces the final path segment unless the base path ends with `/`. + let path = base.path(); + if !path.ends_with('/') { + let mut with_slash = path.to_string(); + with_slash.push('/'); + base.set_path(&with_slash); + } + + match base.join(relative_path) { + Ok(joined) => joined.to_string(), + Err(_) => format!("{}/{}", base_url.trim_end_matches('/'), relative_path), + } +} + impl SingleArchive { /// Returns the total extracted plain-output size for this archive. pub fn output_size(&self) -> u64 { @@ -608,6 +679,7 @@ pub fn generate_manifest( .iter() .map(|files| files.iter().map(|file| file.size).sum()) .collect(), + chunk_files: vec![], chunk_output_files, }), ); @@ -920,6 +992,7 @@ mod tests { total_blocks: 1_500_000, chunk_sizes: vec![80_000, 100_000, 120_000], chunk_decompressed_sizes: vec![], + chunk_files: vec![], chunk_output_files: vec![vec![], vec![], vec![]], }), ); @@ -930,6 +1003,7 @@ mod tests { total_blocks: 1_500_000, chunk_sizes: vec![40_000, 50_000, 60_000], chunk_decompressed_sizes: vec![], + chunk_files: vec![], chunk_output_files: vec![vec![], vec![], vec![]], }), ); @@ -1040,6 +1114,7 @@ mod tests { total_blocks: 1_005_000, chunk_sizes: vec![10, 20, 30], chunk_decompressed_sizes: vec![100, 200, 300], + chunk_files: vec![], chunk_output_files: vec![vec![], vec![], vec![]], }), ); @@ -1061,6 +1136,7 @@ mod tests { total_blocks: 1_005_000, chunk_sizes: vec![10, 20, 30], chunk_decompressed_sizes: vec![], + chunk_files: vec![], chunk_output_files: vec![], }; // The final chunk holds only 5_000 blocks, so covering 10_064 blocks needs the last two @@ -1130,6 +1206,7 @@ mod tests { total_blocks: 24_396_822, chunk_sizes: vec![100; 49], // 49 chunks chunk_decompressed_sizes: vec![], + chunk_files: vec![], chunk_output_files: vec![vec![]; 49], }), ); @@ -1199,6 +1276,7 @@ mod tests { total_blocks: 1_000_000, chunk_sizes: vec![80_000, 120_000], chunk_decompressed_sizes: vec![111, 222], + chunk_files: vec![], chunk_output_files: vec![ vec![OutputFileChecksum { path: "static_files/static_file_transactions_0_499999.bin".to_string(), @@ -1258,6 +1336,7 @@ mod tests { total_blocks: 1_000_000, chunk_sizes: vec![80_000, 120_000], chunk_decompressed_sizes: vec![111, 222], + chunk_files: vec![], chunk_output_files: vec![ vec![OutputFileChecksum { path: "static_files/static_file_transactions_0_499999.bin".to_string(), @@ -1342,4 +1421,160 @@ mod tests { assert_eq!(rocksdb.output_files[0].path, "rocksdb/CURRENT"); assert!(output.path().join("rocksdb_indices.tar.zst").exists()); } + + #[test] + fn resolve_archive_url_joins_nested_paths_under_base() { + assert_eq!( + resolve_archive_url( + "https://example.com/mainnet", + "static_files/headers-0-499999.tar.zst" + ), + "https://example.com/mainnet/static_files/headers-0-499999.tar.zst" + ); + assert_eq!( + resolve_archive_url("https://example.com/mainnet/", "1700000/state.tar.zst"), + "https://example.com/mainnet/1700000/state.tar.zst" + ); + assert_eq!( + resolve_archive_url("https://example.com/mainnet", "headers-0-499999.tar.zst"), + "https://example.com/mainnet/headers-0-499999.tar.zst" + ); + } + + #[test] + fn chunk_files_resolve_relative_to_root_base_url() { + let mut components = BTreeMap::new(); + components.insert( + "headers".to_string(), + ComponentManifest::Chunked(ChunkedArchive { + blocks_per_file: 500_000, + total_blocks: 1_000_000, + chunk_sizes: vec![40_000, 50_000], + chunk_decompressed_sizes: vec![], + chunk_files: vec![ + "static_files/headers-0-499999.tar.zst".to_string(), + "1700000/headers-500000-999999.tar.zst".to_string(), + ], + chunk_output_files: vec![vec![], vec![]], + }), + ); + components.insert( + "state".to_string(), + ComponentManifest::Single(SingleArchive { + file: "1700000/state.tar.zst".to_string(), + size: 100, + decompressed_size: 0, + blake3: None, + output_files: vec![], + }), + ); + + let m = SnapshotManifest { + block: 1_000_000, + chain_id: 1, + storage_version: 2, + timestamp: 1_700_000, + base_url: Some("https://example.com/mainnet".to_string()), + reth_version: None, + components, + }; + + let urls = m.archive_urls(SnapshotComponentType::Headers); + assert_eq!(urls.len(), 2, "exactly 2 header chunk URLs"); + assert_eq!( + urls[0], "https://example.com/mainnet/static_files/headers-0-499999.tar.zst", + "finalized chunk stays under static_files/" + ); + assert_eq!( + urls[1], "https://example.com/mainnet/1700000/headers-500000-999999.tar.zst", + "tip chunk resolves under the run timestamp directory" + ); + + let state = m.snapshot_archives_for_distance(SnapshotComponentType::State, None); + assert_eq!(state.len(), 1, "exactly one state archive"); + assert_eq!( + state[0].url, "https://example.com/mainnet/1700000/state.tar.zst", + "single archive file path joins under root base_url without ../" + ); + assert_eq!(state[0].file_name, "1700000/state.tar.zst"); + + let headers = + m.snapshot_archives_for_distance(SnapshotComponentType::Headers, Some(500_000)); + assert_eq!(headers.len(), 1, "distance selection returns only the tip chunk"); + assert_eq!( + headers[0].file_name, "1700000/headers-500000-999999.tar.zst", + "file_name keeps the relative path from chunk_files" + ); + assert_eq!( + headers[0].url, + "https://example.com/mainnet/1700000/headers-500000-999999.tar.zst" + ); + } + + #[test] + fn chunk_files_absent_keeps_default_chunk_names() { + let m = test_manifest(); + let archives = m.snapshot_archives_for_distance(SnapshotComponentType::Transactions, None); + assert_eq!(archives.len(), 3); + assert_eq!(archives[0].file_name, "transactions-0-499999.tar.zst"); + assert_eq!(archives[0].url, "https://example.com/transactions-0-499999.tar.zst"); + } + + #[test] + fn resolve_archive_url_normalizes_parent_dirs_and_leading_slashes() { + assert_eq!( + resolve_archive_url("https://example.com/static_files", "../1700000/state.tar.zst"), + "https://example.com/1700000/state.tar.zst", + "../ under a static_files base resolves to the run directory" + ); + assert_eq!( + resolve_archive_url( + "https://example.com/mainnet", + "/static_files/headers-0-499999.tar.zst" + ), + "https://example.com/mainnet/static_files/headers-0-499999.tar.zst", + "leading slash is stripped so the base path prefix is preserved" + ); + } + + #[test] + fn mismatched_chunk_files_length_falls_back_to_default_names() { + let mut components = BTreeMap::new(); + components.insert( + "headers".to_string(), + ComponentManifest::Chunked(ChunkedArchive { + blocks_per_file: 500_000, + total_blocks: 1_000_000, + chunk_sizes: vec![40_000, 50_000], + chunk_decompressed_sizes: vec![], + // Only one entry for two chunks — ignored in favor of default names. + chunk_files: vec!["static_files/headers-0-499999.tar.zst".to_string()], + chunk_output_files: vec![vec![], vec![]], + }), + ); + let m = SnapshotManifest { + block: 1_000_000, + chain_id: 1, + storage_version: 2, + timestamp: 0, + base_url: Some("https://example.com/mainnet".to_string()), + reth_version: None, + components, + }; + + let ComponentManifest::Chunked(chunked) = + m.component(SnapshotComponentType::Headers).unwrap() + else { + panic!("headers should be chunked"); + }; + assert!(!chunked.chunk_files_are_consistent()); + + let urls = m.archive_urls(SnapshotComponentType::Headers); + assert_eq!(urls.len(), 2); + assert_eq!( + urls[0], "https://example.com/mainnet/headers-0-499999.tar.zst", + "mismatched chunk_files must not partially apply" + ); + assert_eq!(urls[1], "https://example.com/mainnet/headers-500000-999999.tar.zst"); + } } diff --git a/crates/cli/commands/src/download/planning.rs b/crates/cli/commands/src/download/planning.rs index 1072a00b666..f087926d7d0 100644 --- a/crates/cli/commands/src/download/planning.rs +++ b/crates/cli/commands/src/download/planning.rs @@ -204,6 +204,18 @@ pub(crate) fn collect_planned_archives( let Some(distance) = selection_archive_distance(selection, manifest.block) else { continue; }; + + if let Some(ComponentManifest::Chunked(chunked)) = manifest.component(*ty) && + !chunked.chunk_files_are_consistent() + { + eyre::bail!( + "Invalid modular manifest: {} chunk_files length ({}) does not match chunk count ({})", + ty.key(), + chunked.chunk_files.len(), + chunked.num_chunks() + ); + } + total_download_size += manifest.size_for_distance(*ty, distance); total_output_size += manifest.output_size_for_distance(*ty, distance); @@ -382,6 +394,7 @@ mod tests { total_blocks: 1_000_000, chunk_sizes: vec![20, 30], chunk_decompressed_sizes: vec![200, 300], + chunk_files: vec![], chunk_output_files: vec![ vec![OutputFileChecksum { path: "static_files/tx-0".to_string(), @@ -446,4 +459,49 @@ mod tests { }) ); } + + #[test] + fn collect_planned_archives_rejects_mismatched_chunk_files_length() { + let mut components = BTreeMap::new(); + components.insert( + "transactions".to_string(), + ComponentManifest::Chunked(ChunkedArchive { + blocks_per_file: 500_000, + total_blocks: 1_000_000, + chunk_sizes: vec![20, 30], + chunk_decompressed_sizes: vec![200, 300], + chunk_files: vec!["static_files/transactions-0-499999.tar.zst".to_string()], + chunk_output_files: vec![ + vec![OutputFileChecksum { + path: "static_files/tx-0".to_string(), + size: 200, + blake3: "h1".to_string(), + }], + vec![OutputFileChecksum { + path: "static_files/tx-1".to_string(), + size: 300, + blake3: "h2".to_string(), + }], + ], + }), + ); + + let manifest = SnapshotManifest { + block: 1_000_000, + chain_id: 1, + storage_version: 2, + timestamp: 0, + base_url: Some("https://example.com/mainnet".to_string()), + reth_version: None, + components, + }; + let selections = + BTreeMap::from([(SnapshotComponentType::Transactions, ComponentSelection::All)]); + + let err = collect_planned_archives(&manifest, &selections).unwrap_err(); + assert!( + err.to_string().contains("chunk_files length"), + "expected chunk_files length error, got: {err}" + ); + } } From 53ee0ac63dcf8d185e76c4eb038c8170c0abb552 Mon Sep 17 00:00:00 2001 From: Chase Wright Date: Mon, 10 Aug 2026 10:27:35 -0500 Subject: [PATCH 114/148] fix(rpc): honor targetGasLimit in testing_buildBlockV1 (#26632) --- crates/ethereum/node/tests/it/testing.rs | 7 ++++++- crates/rpc/rpc/src/testing.rs | 8 +++++++- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/crates/ethereum/node/tests/it/testing.rs b/crates/ethereum/node/tests/it/testing.rs index 4a3740d9882..df483211b9a 100644 --- a/crates/ethereum/node/tests/it/testing.rs +++ b/crates/ethereum/node/tests/it/testing.rs @@ -133,6 +133,7 @@ async fn testing_rpc_commit_block_works() -> eyre::Result<()> { let chain = ctx.config().chain.clone(); let timestamp = chain.genesis().timestamp + 1; + let target_gas_limit = chain.genesis().gas_limit - 100; let payload_attributes = EthPayloadAttributes { timestamp, prev_randao: B256::ZERO, @@ -140,7 +141,7 @@ async fn testing_rpc_commit_block_works() -> eyre::Result<()> { withdrawals: Some(vec![]), parent_beacon_block_root: Some(B256::ZERO), slot_number: Some(timestamp), - target_gas_limit: None, + target_gas_limit: Some(target_gas_limit), }; tokio::spawn(async move { @@ -163,6 +164,10 @@ async fn testing_rpc_commit_block_works() -> eyre::Result<()> { latest.get("hash").and_then(Value::as_str).expect("latest block hash"); assert_eq!(latest_hash, block_hash.to_string()); assert_eq!(latest.get("extraData"), Some(&serde_json::to_value(extra_data)?)); + assert_eq!( + latest.get("gasLimit").and_then(Value::as_str), + Some(format!("{target_gas_limit:#x}").as_str()) + ); assert!(latest .get("transactions") .and_then(Value::as_array) diff --git a/crates/rpc/rpc/src/testing.rs b/crates/rpc/rpc/src/testing.rs index ad4ae5e14d2..b166f913a3f 100644 --- a/crates/rpc/rpc/src/testing.rs +++ b/crates/rpc/rpc/src/testing.rs @@ -152,7 +152,13 @@ where suggested_fee_recipient: request.payload_attributes.suggested_fee_recipient, prev_randao: request.payload_attributes.prev_randao, gas_limit: gas_limit_override.unwrap_or_else(|| { - calculate_block_gas_limit(parent.gas_limit(), desired_gas_limit) + calculate_block_gas_limit( + parent.gas_limit(), + request + .payload_attributes + .target_gas_limit + .unwrap_or(desired_gas_limit), + ) }), parent_beacon_block_root: request.payload_attributes.parent_beacon_block_root, withdrawals: withdrawals.map(Into::into), From 64e622584f053615d3cb74872f3b528cfee1bdc1 Mon Sep 17 00:00:00 2001 From: joshieDo <93316087+joshieDo@users.noreply.github.com> Date: Mon, 10 Aug 2026 19:32:52 +0100 Subject: [PATCH 115/148] feat(storage): add rocksdb BAL store (#25476) Co-authored-by: Matthias Seitz --- crates/cli/commands/src/common.rs | 16 +- crates/engine/tree/src/persistence.rs | 45 +- crates/net/network/src/eth_requests.rs | 5 +- crates/node/builder/src/launch/common.rs | 19 +- crates/rpc/rpc-engine-api/src/engine_api.rs | 7 +- crates/rpc/rpc-eth-types/src/cache/mod.rs | 32 +- crates/storage/db-api/src/models/bal.rs | 200 ++++++ crates/storage/db-api/src/models/mod.rs | 2 + crates/storage/db-api/src/tables/mod.rs | 13 + crates/storage/provider/src/bal.rs | 7 +- crates/storage/provider/src/bal/rocksdb.rs | 600 ++++++++++++++++++ crates/storage/provider/src/lib.rs | 2 +- .../src/providers/blockchain_provider.rs | 2 +- .../provider/src/providers/database/mod.rs | 2 +- .../provider/src/providers/rocksdb/metrics.rs | 2 + .../src/providers/rocksdb/provider.rs | 106 +++- crates/storage/storage-api/src/bal.rs | 81 +-- 17 files changed, 1062 insertions(+), 79 deletions(-) create mode 100644 crates/storage/db-api/src/models/bal.rs create mode 100644 crates/storage/provider/src/bal/rocksdb.rs diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 2748ba5ec9c..77cd552633f 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -27,8 +27,7 @@ use reth_provider::{ BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider, StaticFileProviderBuilder, }, - BalConfig, BalStoreHandle, InMemoryBalStore, ProviderFactory, StaticFileProviderFactory, - StorageSettings, + BalStoreHandle, ProviderFactory, RocksDBBalStore, StaticFileProviderFactory, StorageSettings, }; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; @@ -196,11 +195,14 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let balstore_cache_size = - self.db.balstore_cache_size.unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); - let bal_store = BalStoreHandle::new(InMemoryBalStore::new( - BalConfig::with_in_memory_retention_distance(balstore_cache_size), - )); + let bal_store = self + .db + .balstore_cache_size + .map(|distance| { + RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) + }) + .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); + let bal_store = BalStoreHandle::new(bal_store); let factory = ProviderFactory::>::new( db, self.chain.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 7f461b9e461..113e7a0f47a 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -176,6 +176,11 @@ where let start_time = Instant::now(); + let canonical_blocks = input + .persist_rest_blocks() + .iter() + .map(|block| block.recovered_block().num_hash()) + .collect::>(); let provider_rw = self.provider.database_provider_rw()?; let last_state_trie_block = if let Some(block) = input.state_trie_blocks().last() { // Newly written static-file headers are not readable until commit finalizes their @@ -206,7 +211,8 @@ where } provider_rw.commit()?; - let _ = self.provider.bal_store().flush().inspect_err(|err| { + // BALs live outside the main database and are intentionally flushed last. + let _ = self.provider.bal_store().flush(&canonical_blocks).inspect_err(|err| { warn!(target: "engine::persistence", last=?last_block, ?err, "Failed to flush BAL store"); }); debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks"); @@ -560,6 +566,43 @@ mod tests { assert_eq!(result.last_state_trie_block, result.last_block); } + #[test] + fn test_save_blocks_flushes_bal_store() { + use reth_provider::{RocksDBBalStore, RocksDBProviderFactory}; + + reth_tracing::init_test_tracing(); + let provider = create_test_provider_factory(); + let bal_store = BalStoreHandle::new(RocksDBBalStore::new(provider.rocksdb_provider())); + let provider = provider.with_bal_store(bal_store); + init_genesis(&provider).unwrap(); + + let mut test_block_builder = TestBlockBuilder::eth(); + let executed = test_block_builder.get_executed_block_with_number(1, B256::random()); + let num_hash = executed.recovered_block().num_hash(); + let raw_bal = Bytes::from_static(&[0xc0]); + + provider.bal_store().insert(num_hash, RawBal::new(raw_bal.clone())).unwrap(); + + let (_finished_exex_height_tx, finished_exex_height_rx) = + tokio::sync::watch::channel(FinishedExExHeight::NoExExs); + let pruner = + Pruner::new_with_factory(provider.clone(), vec![], 5, 0, None, finished_exex_height_rx); + let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); + let handle = PersistenceHandle::::spawn_service( + provider.clone(), + pruner, + sync_metrics_tx, + ); + let (tx, rx) = crossbeam_channel::bounded(1); + + handle.save_blocks(full_save_input(vec![executed]), tx).unwrap(); + + let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out"); + assert_eq!(result.last_block, num_hash); + let persisted_store = RocksDBBalStore::new(provider.rocksdb_provider()); + assert_eq!(persisted_store.get_by_hash(num_hash.hash).unwrap(), Some(raw_bal)); + } + #[test] fn test_save_blocks_multiple_blocks() { reth_tracing::init_test_tracing(); diff --git a/crates/net/network/src/eth_requests.rs b/crates/net/network/src/eth_requests.rs index fbf6d53b924..5d1172e31f8 100644 --- a/crates/net/network/src/eth_requests.rs +++ b/crates/net/network/src/eth_requests.rs @@ -412,7 +412,7 @@ where let limit = GetBlockAccessListLimit::ResponseSizeSoftLimit(SOFT_RESPONSE_LIMIT); let access_lists = - self.client.bal_store().get_by_hashes_with_limit(&request.0, limit).unwrap_or_default(); + self.client.get_bals_by_hashes_with_limit(&request.0, limit).unwrap_or_default(); let _ = response.send(Ok(BlockAccessLists(access_lists))); } } @@ -470,8 +470,7 @@ where ); let block_access_lists = self .client - .bal_store() - .get_by_hashes_with_limit(&req.block_hashes, limit) + .get_bals_by_hashes_with_limit(&req.block_hashes, limit) .unwrap_or_default(); Ok(SnapResponse::BlockAccessLists(BlockAccessListsMessage { request_id: req.request_id, diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 498c65b3fab..43b6cc93bfc 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -71,10 +71,10 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, - BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, - DatabaseProviderFactory, InMemoryBalStore, MetadataProvider, MetadataWriter, ProviderError, - ProviderFactory, ProviderResult, RocksDBProviderFactory, StageCheckpointReader, - StaticFileProviderBuilder, StaticFileProviderFactory, StorageSettingsCache, + BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, DatabaseProviderFactory, + MetadataProvider, MetadataWriter, ProviderError, ProviderFactory, ProviderResult, + RocksDBBalStore, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder, + StaticFileProviderFactory, StorageSettingsCache, }; use reth_prune::{PruneMode, PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; @@ -524,14 +524,15 @@ where .build()? }; - let balstore_cache_size = self + let bal_store = self .node_config() .db .balstore_cache_size - .unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); - let bal_store = BalStoreHandle::new(InMemoryBalStore::new( - BalConfig::with_in_memory_retention_distance(balstore_cache_size), - )); + .map(|distance| { + RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) + }) + .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); + let bal_store = BalStoreHandle::new(bal_store); let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), diff --git a/crates/rpc/rpc-engine-api/src/engine_api.rs b/crates/rpc/rpc-engine-api/src/engine_api.rs index ab08df5e5ab..8053becc1f6 100644 --- a/crates/rpc/rpc-engine-api/src/engine_api.rs +++ b/crates/rpc/rpc-engine-api/src/engine_api.rs @@ -817,7 +817,7 @@ where } let (tx, rx) = oneshot::channel(); - let bal_store = self.inner.provider.bal_store().clone(); + let inner = self.inner.clone(); self.inner.task_spawner.spawn_blocking_task(async move { if tx.is_closed() { @@ -825,8 +825,9 @@ where } tx.send( - bal_store - .get_by_hashes(&hashes) + inner + .provider + .get_bals_by_hashes(&hashes) .map_err(|err| EngineApiError::Internal(Box::new(err))), ) .ok(); diff --git a/crates/rpc/rpc-eth-types/src/cache/mod.rs b/crates/rpc/rpc-eth-types/src/cache/mod.rs index 40a186e234d..cdbb33e4a1f 100644 --- a/crates/rpc/rpc-eth-types/src/cache/mod.rs +++ b/crates/rpc/rpc-eth-types/src/cache/mod.rs @@ -5,7 +5,7 @@ use crate::block::CachedTransaction; use alloy_consensus::{transaction::TxHashRef, BlockHeader}; use alloy_eip7928::bal::DecodedBal; use alloy_eips::BlockHashOrNumber; -use alloy_primitives::{Address, TxHash, B256}; +use alloy_primitives::{Address, Bytes, TxHash, B256}; use futures::{stream::FuturesOrdered, Stream, StreamExt}; use reth_chain_state::CanonStateNotification; use reth_errors::{ProviderError, ProviderResult}; @@ -625,10 +625,11 @@ where ActionSender::new(CacheKind::Bal, block_hash, action_tx); this.action_task_spawner.spawn_blocking_task(async move { let _permit = rate_limiter.acquire().await; - let res = provider - .bal_store() - .revm_bal_by_hash(block_hash) - .map(|maybe_bal| maybe_bal.map(CachedRevmBal::new)); + let res = provider.get_bal_by_hash(block_hash).and_then( + |maybe_bal| { + maybe_bal.map(CachedRevmBal::try_from_raw).transpose() + }, + ); action_sender.send_bal(res); }); } @@ -944,6 +945,18 @@ impl CachedRevmBal { fn new(bal: DecodedBal>) -> Self { Self(Arc::new(bal)) } + + /// Decodes raw BAL bytes into the representation used by revm. + fn try_from_raw(raw: Bytes) -> ProviderResult { + DecodedBal::from_rlp_bytes(raw) + .map_err(Into::into) + .and_then(|decoded| { + decoded.try_map(|bal| { + RevmBal::try_from(Vec::from(bal)).map(Arc::new).map_err(ProviderError::other) + }) + }) + .map(Self::new) + } } impl InMemorySize for CachedRevmBal { @@ -1237,15 +1250,8 @@ mod tests { } fn get_by_hashes(&self, block_hashes: &[BlockHash]) -> ProviderResult>> { - Ok(block_hashes.iter().map(|_| None).collect()) - } - - fn revm_bal_by_hash( - &self, - _block_hash: BlockHash, - ) -> ProviderResult>>> { self.fetches.fetch_add(1, Ordering::SeqCst); - Ok(Some(test_decoded_revm_bal())) + Ok(block_hashes.iter().map(|_| Some(Bytes::from_static(&[0xc0]))).collect()) } fn bal_stream(&self) -> reth_storage_api::BalNotificationStream { diff --git a/crates/storage/db-api/src/models/bal.rs b/crates/storage/db-api/src/models/bal.rs new file mode 100644 index 00000000000..2ca01c95f9c --- /dev/null +++ b/crates/storage/db-api/src/models/bal.rs @@ -0,0 +1,200 @@ +//! Block access list table models. + +use crate::{ + table::{Compress, Decode, Decompress, Encode}, + DatabaseError, +}; +use alloy_primitives::{keccak256, BlockNumber, Bytes, B256}; +use bytes::BufMut; +use core::cmp::Ordering; +use reth_codecs::DecompressError; +use serde::{Deserialize, Serialize}; + +/// Encoded [`StoredBlockAccessListKey`] length. +const BLOCK_ACCESS_LIST_KEY_BYTES: usize = 8 + 32; + +/// Hash prefix length in [`StoredBlockAccessList`] values. +const STORED_BLOCK_ACCESS_LIST_HASH_BYTES: usize = 32; + +/// Block access list table key. +/// +/// Encoded as block number followed by block hash so pruning can scan by block number. +#[derive(Debug, Default, Copy, Clone, PartialEq, Eq, Serialize, Deserialize, Hash)] +pub struct StoredBlockAccessListKey { + block_number: BlockNumber, + block_hash: B256, +} + +impl StoredBlockAccessListKey { + /// Creates a key from a block number/hash pair. + pub const fn new(block_number: BlockNumber, block_hash: B256) -> Self { + Self { block_number, block_hash } + } + + /// Returns the smallest key for the given block number. + pub const fn first_at_number(block_number: BlockNumber) -> Self { + Self::new(block_number, B256::ZERO) + } + + /// Returns the block number. + pub const fn number(&self) -> BlockNumber { + self.block_number + } + + /// Returns the block hash. + pub const fn hash(&self) -> B256 { + self.block_hash + } +} + +impl Ord for StoredBlockAccessListKey { + fn cmp(&self, other: &Self) -> Ordering { + self.block_number + .cmp(&other.block_number) + .then_with(|| self.block_hash.as_slice().cmp(other.block_hash.as_slice())) + } +} + +impl PartialOrd for StoredBlockAccessListKey { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Encode for StoredBlockAccessListKey { + type Encoded = [u8; BLOCK_ACCESS_LIST_KEY_BYTES]; + + fn encode(self) -> Self::Encoded { + let mut buf = [0u8; BLOCK_ACCESS_LIST_KEY_BYTES]; + buf[..8].copy_from_slice(&self.block_number.to_be_bytes()); + buf[8..].copy_from_slice(self.block_hash.as_slice()); + buf + } +} + +impl Decode for StoredBlockAccessListKey { + fn decode(value: &[u8]) -> Result { + if value.len() != BLOCK_ACCESS_LIST_KEY_BYTES { + return Err(DatabaseError::Decode) + } + + let block_number = + u64::from_be_bytes(value[..8].try_into().map_err(|_| DatabaseError::Decode)?); + let block_hash = B256::decode(&value[8..])?; + + Ok(Self::new(block_number, block_hash)) + } +} + +/// Stored block access list value. +#[derive(Debug, Clone, PartialEq, Eq, Serialize)] +pub struct StoredBlockAccessList { + /// Keccak hash carried by the source BAL, trusted without verification on decode. + hash: B256, + /// Raw BAL RLP bytes. + raw: Bytes, +} + +impl StoredBlockAccessList { + /// Creates a stored BAL from raw bytes. + pub fn new(raw: Bytes) -> Self { + let hash = keccak256(&raw); + Self::new_unchecked(hash, raw) + } + + /// Creates a stored BAL from its hash and raw bytes without verifying that they match. + pub const fn new_unchecked(hash: B256, raw: Bytes) -> Self { + Self { hash, raw } + } + + /// Returns the stored hash without verifying it against the raw bytes. + pub const fn hash(&self) -> B256 { + self.hash + } + + /// Consumes the stored BAL and returns its raw bytes. + pub fn into_raw(self) -> Bytes { + self.raw + } +} + +impl Compress for StoredBlockAccessList { + type Compressed = Vec; + + fn compress(self) -> Self::Compressed { + let mut out = Vec::with_capacity(STORED_BLOCK_ACCESS_LIST_HASH_BYTES + self.raw.len()); + out.extend_from_slice(self.hash.as_slice()); + out.extend_from_slice(&self.raw); + out + } + + fn compress_to_buf>(&self, buf: &mut B) { + buf.put_slice(self.hash.as_slice()); + buf.put_slice(&self.raw); + } +} + +impl Decompress for StoredBlockAccessList { + fn decompress(value: &[u8]) -> Result { + if value.len() < STORED_BLOCK_ACCESS_LIST_HASH_BYTES { + return Err(DecompressError::new(StoredBlockAccessListDecodeError)) + } + + let hash = B256::from_slice(&value[..STORED_BLOCK_ACCESS_LIST_HASH_BYTES]); + let raw = Bytes::copy_from_slice(&value[STORED_BLOCK_ACCESS_LIST_HASH_BYTES..]); + + Ok(Self::new_unchecked(hash, raw)) + } +} + +/// Error returned when a stored BAL value is too short to contain its hash prefix. +#[derive(Debug, derive_more::Display, derive_more::Error)] +#[display("stored block access list value is missing its hash prefix")] +struct StoredBlockAccessListDecodeError; + +#[cfg(test)] +mod tests { + use super::*; + use crate::table::{Compress, Decompress}; + + #[test] + fn key_encodes_number_first() { + let low_hash = B256::with_last_byte(0xff); + let high_hash = B256::ZERO; + let low_number = StoredBlockAccessListKey::new(1, low_hash).encode(); + let high_number = StoredBlockAccessListKey::new(2, high_hash).encode(); + + assert!(low_number < high_number); + } + + #[test] + fn key_roundtrip() { + let key = StoredBlockAccessListKey::new(42, B256::with_last_byte(7)); + let encoded = key.encode(); + + assert_eq!(StoredBlockAccessListKey::decode(&encoded).unwrap(), key); + } + + #[test] + fn stored_bal_roundtrip() { + let raw = Bytes::from_static(&[0xc0]); + let stored = StoredBlockAccessList::new(raw.clone()); + let encoded = stored.clone().compress(); + let decoded = StoredBlockAccessList::decompress(&encoded).unwrap(); + + assert_eq!(decoded, stored); + assert_eq!(decoded.hash(), keccak256(&raw)); + assert_eq!(decoded.into_raw(), raw); + } + + #[test] + fn stored_bal_unchecked_preserves_hash_and_raw_bytes() { + let hash = B256::with_last_byte(1); + let raw = Bytes::from_static(&[0xc0]); + let stored = StoredBlockAccessList::new_unchecked(hash, raw.clone()); + let encoded = stored.compress(); + + assert_eq!(&encoded[..STORED_BLOCK_ACCESS_LIST_HASH_BYTES], hash.as_slice()); + assert_eq!(&encoded[STORED_BLOCK_ACCESS_LIST_HASH_BYTES..], raw.as_ref()); + } +} diff --git a/crates/storage/db-api/src/models/mod.rs b/crates/storage/db-api/src/models/mod.rs index ec7cd534278..aa7717252b1 100644 --- a/crates/storage/db-api/src/models/mod.rs +++ b/crates/storage/db-api/src/models/mod.rs @@ -11,6 +11,7 @@ use reth_trie_common::{StoredNibbles, StoredNibblesSubKey, *}; use serde::{Deserialize, Serialize}; pub mod accounts; +pub mod bal; pub mod blocks; pub mod integer_list; pub mod metadata; @@ -18,6 +19,7 @@ pub mod sharded_key; pub mod storage_sharded_key; pub use accounts::*; +pub use bal::*; pub use blocks::*; pub use integer_list::IntegerList; pub use metadata::*; diff --git a/crates/storage/db-api/src/tables/mod.rs b/crates/storage/db-api/src/tables/mod.rs index 1e90fd66a21..407e2f9ad69 100644 --- a/crates/storage/db-api/src/tables/mod.rs +++ b/crates/storage/db-api/src/tables/mod.rs @@ -19,6 +19,7 @@ pub use raw::{RawDupSort, RawKey, RawTable, RawValue, TableRawRow}; use crate::{ models::{ accounts::BlockNumberAddress, + bal::{StoredBlockAccessList, StoredBlockAccessListKey}, blocks::{HeaderHash, StoredBlockOmmers}, storage_sharded_key::StorageShardedKey, AccountBeforeTx, ClientVersion, CompactU256, IntegerList, ShardedKey, @@ -351,6 +352,18 @@ tables! { type Value = StoredBlockWithdrawals; } + /// Stores block access list payloads by block number and block hash. + table BlockAccessLists { + type Key = StoredBlockAccessListKey; + type Value = StoredBlockAccessList; + } + + /// Stores the block number for each persisted block access list hash. + table BlockAccessListBlockNumbers { + type Key = BlockHash; + type Value = BlockNumber; + } + /// Canonical only Stores the transaction body for canonical transactions. table Transactions { type Key = TxNumber; diff --git a/crates/storage/provider/src/bal.rs b/crates/storage/provider/src/bal.rs index 45b405cd5fe..8ffc049d82e 100644 --- a/crates/storage/provider/src/bal.rs +++ b/crates/storage/provider/src/bal.rs @@ -13,6 +13,9 @@ use std::{ sync::Arc, }; +mod rocksdb; +pub use rocksdb::RocksDBBalStore; + /// Basic in-memory BAL store keyed by block hash. #[derive(Debug, Clone)] pub struct InMemoryBalStore { @@ -161,7 +164,7 @@ impl BalStore for InMemoryBalStore { Ok(()) } - fn flush(&self) -> ProviderResult<()> { + fn flush(&self, _blocks: &[NumHash]) -> ProviderResult<()> { Ok(()) } @@ -250,7 +253,7 @@ mod tests { fn flush_is_noop() { let store = InMemoryBalStore::default(); - store.flush().unwrap(); + store.flush(&[]).unwrap(); } #[test] diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs new file mode 100644 index 00000000000..d5886d1e8d9 --- /dev/null +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -0,0 +1,600 @@ +use crate::providers::RocksDBProvider; +use alloy_eip7928::BAL_RETENTION_PERIOD_SLOTS; +use alloy_eips::NumHash; +use alloy_primitives::{BlockHash, BlockNumber, Bytes}; +use parking_lot::RwLock; +use reth_db_api::{ + models::{StoredBlockAccessList, StoredBlockAccessListKey}, + table::{Decode, Decompress}, + tables, DatabaseError, +}; +use reth_prune_types::PruneMode; +use reth_storage_api::{ + BalNotification, BalNotificationStream, BalStore, GetBlockAccessListLimit, RawBal, +}; +use reth_storage_errors::provider::{ProviderError, ProviderResult}; +use reth_tokio_util::EventSender; +use std::{ + collections::{BTreeMap, BTreeSet, HashMap}, + sync::Arc, +}; + +/// Number of recent blocks kept in the in-memory `RocksDB` BAL buffer. +const DEFAULT_BAL_BUFFER_RETENTION_DISTANCE: u64 = 32; + +/// RocksDB-backed BAL store. +/// +/// Persisted BALs are keyed by `(block_number, block_hash)` for ordered pruning and indexed by +/// block hash for direct [`BalStore`] lookups. Validated BALs enter a shared in-memory buffer +/// first; [`BalStore::flush`] makes confirmed canonical entries durable while retaining them in the +/// read cache until its shorter retention window expires. +#[derive(Clone)] +pub struct RocksDBBalStore { + /// Number of recent blocks retained in the in-memory buffer. + buffer_retention_distance: u64, + /// `RocksDB` provider used for persisted BAL reads and writes. + rocksdb: RocksDBProvider, + /// Shared recent-read cache and pending-write state. + buffer: Arc>, + /// Broadcasts BAL insert notifications. + notifications: EventSender, +} + +impl RocksDBBalStore { + /// Creates a new store with the EIP-defined retention distance. + pub fn new(rocksdb: RocksDBProvider) -> Self { + Self::with_buffer_retention_distance(rocksdb, DEFAULT_BAL_BUFFER_RETENTION_DISTANCE) + } + + /// Creates a new store that retains buffered BALs for the given block distance. + /// + /// This does not change the EIP-defined retention distance for persisted BALs. + pub fn with_buffer_retention_distance(rocksdb: RocksDBProvider, blocks: u64) -> Self { + Self { + buffer_retention_distance: blocks, + rocksdb, + buffer: Arc::new(RwLock::new(RocksDBBalStoreBuffer::default())), + notifications: EventSender::new(super::DEFAULT_BAL_NOTIFICATION_CHANNEL_SIZE), + } + } + + #[cfg(test)] + const fn rocksdb_provider(&self) -> &RocksDBProvider { + &self.rocksdb + } + + fn keys_to_prune(&self, tip: BlockNumber) -> ProviderResult> { + let retention = PruneMode::Distance(BAL_RETENTION_PERIOD_SLOTS); + let mut keys = Vec::new(); + let iter = self.rocksdb.raw_key_iter_from::( + StoredBlockAccessListKey::first_at_number(0), + )?; + + for key_bytes in iter { + let key_bytes = key_bytes?; + let key = StoredBlockAccessListKey::decode(&key_bytes) + .map_err(|_| ProviderError::Database(DatabaseError::Decode))?; + if !retention.should_prune(key.number(), tip) { + break + } + keys.push(key); + } + + Ok(keys) + } + + fn delete_keys(&self, keys: &[StoredBlockAccessListKey]) -> ProviderResult { + if keys.is_empty() { + return Ok(0) + } + + let mut batch = self.rocksdb.batch(); + for key in keys { + batch.delete::(*key)?; + batch.delete::(key.hash())?; + } + batch.commit()?; + Ok(keys.len()) + } + + fn read_one_from_disk(&self, key: StoredBlockAccessListKey) -> ProviderResult> { + let Some(value) = self.rocksdb.get_raw::(key)? else { + return Ok(None) + }; + let stored = StoredBlockAccessList::decompress(&value) + .map_err(|_| ProviderError::Database(DatabaseError::Decode))?; + Ok(Some(stored.into_raw())) + } + + fn read_one_by_hash(&self, block_hash: BlockHash) -> ProviderResult> { + if let Some(bal) = self.buffer.read().get_by_hash(block_hash) { + return Ok(Some(bal)) + } + + let Some(block_number) = + self.rocksdb.get::(block_hash)? + else { + return Ok(None) + }; + self.read_one_from_disk(StoredBlockAccessListKey::new(block_number, block_hash)) + } +} + +impl std::fmt::Debug for RocksDBBalStore { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("RocksDBBalStore") + .field("buffer_retention_distance", &self.buffer_retention_distance) + .field("rocksdb", &self.rocksdb) + .finish_non_exhaustive() + } +} + +impl BalStore for RocksDBBalStore { + fn insert(&self, block: NumHash, bal: RawBal) -> ProviderResult<()> { + let mut buffer = self.buffer.write(); + buffer.insert(block, bal.clone()); + drop(buffer); + + self.notifications.notify(BalNotification::new(block, bal)); + Ok(()) + } + + fn insert_many(&self, entries: Vec<(NumHash, RawBal)>) -> ProviderResult<()> { + if entries.is_empty() { + return Ok(()) + } + + let mut buffer = self.buffer.write(); + buffer.entries.reserve(entries.len()); + for (block, bal) in &entries { + buffer.insert(*block, bal.clone()); + } + drop(buffer); + + for (block, bal) in entries { + self.notifications.notify(BalNotification::new(block, bal)); + } + Ok(()) + } + + fn flush(&self, blocks: &[NumHash]) -> ProviderResult<()> { + let pending = { + let mut buffer = self.buffer.write(); + buffer.mark_canonical(blocks); + buffer.canonical_pending_entries() + }; + if !pending.is_empty() { + let mut batch = self.rocksdb.batch(); + for (key, bal) in &pending { + let value = StoredBlockAccessList::new_unchecked(bal.hash(), bal.as_raw().clone()); + batch.put::(*key, &value)?; + batch.put::(key.hash(), &key.number())?; + } + batch.commit()?; + + self.buffer.write().remove_flushed_pending(&pending); + } + + if let Some(tip) = blocks.iter().map(|block| block.number).max() { + self.buffer.write().prune(self.buffer_retention_distance, tip); + } + Ok(()) + } + + fn prune(&self, tip: BlockNumber) -> ProviderResult { + let keys = self.keys_to_prune(tip)?; + let pruned = self.delete_keys(&keys)?; + self.buffer.write().remove_keys(&keys); + Ok(pruned) + } + + fn get_by_hashes(&self, block_hashes: &[BlockHash]) -> ProviderResult>> { + block_hashes.iter().map(|hash| self.read_one_by_hash(*hash)).collect() + } + + fn append_by_hashes_with_limit( + &self, + block_hashes: &[BlockHash], + limit: GetBlockAccessListLimit, + out: &mut Vec>, + ) -> ProviderResult<()> { + let mut size = 0; + for block_hash in block_hashes { + let bal = self.read_one_by_hash(*block_hash)?; + size += bal.as_ref().map_or(1, |bytes| bytes.len()); + out.push(bal); + + if limit.exceeds(size) { + break + } + } + Ok(()) + } + + fn bal_stream(&self) -> BalNotificationStream { + self.notifications.new_listener() + } +} + +/// Shared in-memory state for recent reads and writes awaiting canonical confirmation. +/// +/// Successful flushes clear only the pending-write state. Cached entries remain available until +/// the buffer retention window evicts them. +#[derive(Debug, Default)] +struct RocksDBBalStoreBuffer { + /// Hash index for serving recent hash-only lookups. + entries: HashMap, + /// Block-number index for pruning buffered entries. + hashes_by_number: BTreeMap>, + /// Validated BALs waiting to be confirmed canonical and flushed. + pending: BTreeMap, + /// Pending writes confirmed canonical, including writes retained for retry after failure. + canonical_pending: BTreeSet, +} + +impl RocksDBBalStoreBuffer { + fn insert(&mut self, block: NumHash, bal: RawBal) { + let pending = bal.clone(); + if let Some(entry) = + self.entries.insert(block.hash, RocksDBBalEntry { block_number: block.number, bal }) + { + self.remove_hash_from_number(entry.block_number, block.hash); + self.pending.remove(&StoredBlockAccessListKey::new(entry.block_number, block.hash)); + self.canonical_pending + .remove(&StoredBlockAccessListKey::new(entry.block_number, block.hash)); + } + + self.hashes_by_number.entry(block.number).or_default().push(block.hash); + self.pending.insert(StoredBlockAccessListKey::new(block.number, block.hash), pending); + } + + /// Marks exact pending block identities as eligible for the next flush. + fn mark_canonical(&mut self, blocks: &[NumHash]) { + self.canonical_pending.extend( + blocks + .iter() + .map(|block| StoredBlockAccessListKey::new(block.number, block.hash)) + .filter(|key| self.pending.contains_key(key)), + ); + } + + /// Snapshots confirmed writes so `RocksDB` I/O can run without holding the buffer lock. + fn canonical_pending_entries(&self) -> Vec<(StoredBlockAccessListKey, RawBal)> { + self.canonical_pending + .iter() + .filter_map(|key| self.pending.get(key).map(|bal| (*key, bal.clone()))) + .collect() + } + + fn keys_to_prune( + &self, + retention_distance: u64, + tip: BlockNumber, + ) -> Vec { + let prune_mode = PruneMode::Distance(retention_distance); + self.hashes_by_number + .iter() + .take_while(|(block_number, _)| prune_mode.should_prune(**block_number, tip)) + .flat_map(|(block_number, hashes)| { + hashes.iter().map(move |hash| StoredBlockAccessListKey::new(*block_number, *hash)) + }) + .filter(|key| !self.canonical_pending.contains(key)) + .collect() + } + + fn get_by_hash(&self, hash: BlockHash) -> Option { + self.entries.get(&hash).map(|entry| entry.bal.as_raw().clone()) + } + + /// Clears pending state only if it still matches the snapshot written to `RocksDB`. + /// + /// A concurrent replacement for the same key must remain pending for a later flush. + fn remove_flushed_pending(&mut self, flushed: &[(StoredBlockAccessListKey, RawBal)]) { + for (key, bal) in flushed { + let pending_matches = + self.pending.get(key).is_some_and(|pending| pending.as_raw() == bal.as_raw()); + if pending_matches { + self.pending.remove(key); + self.canonical_pending.remove(key); + } + } + } + + fn prune(&mut self, retention_distance: u64, tip: BlockNumber) { + let keys = self.keys_to_prune(retention_distance, tip); + self.remove_keys(&keys); + } + + fn remove_keys(&mut self, keys: &[StoredBlockAccessListKey]) -> usize { + let mut removed = 0; + for key in keys { + let block = NumHash::new(key.number(), key.hash()); + let pending_removed = self.pending.remove(key).is_some(); + self.canonical_pending.remove(key); + let entry_removed = if self + .entries + .get(&block.hash) + .is_some_and(|entry| entry.block_number == block.number) + { + self.entries.remove(&block.hash).is_some() + } else { + false + }; + + if entry_removed { + self.remove_hash_from_number(block.number, block.hash); + } + removed += usize::from(pending_removed || entry_removed); + } + removed + } + + fn remove_hash_from_number(&mut self, block_number: BlockNumber, block_hash: BlockHash) { + let empty = self.hashes_by_number.get_mut(&block_number).is_some_and(|hashes| { + hashes.retain(|hash| *hash != block_hash); + hashes.is_empty() + }); + if empty { + self.hashes_by_number.remove(&block_number); + } + } +} + +/// Buffered BAL entry with its block number. +#[derive(Debug)] +struct RocksDBBalEntry { + /// Block number for this hash-indexed BAL. + block_number: BlockNumber, + /// Raw BAL payload. + bal: RawBal, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::providers::RocksDBBuilder; + use alloy_primitives::B256; + use tokio_stream::StreamExt; + + fn test_store() -> (tempfile::TempDir, RocksDBBalStore) { + let dir = tempfile::tempdir().unwrap(); + let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); + (dir, RocksDBBalStore::new(rocksdb)) + } + + fn disk_bal(store: &RocksDBBalStore, block: NumHash) -> Option { + store + .rocksdb_provider() + .get_raw::(StoredBlockAccessListKey::new( + block.number, + block.hash, + )) + .unwrap() + .map(|value| StoredBlockAccessList::decompress(&value).unwrap().into_raw()) + } + + fn read_many(store: &RocksDBBalStore, blocks: &[NumHash]) -> Vec> { + blocks.iter().map(|block| store.get_by_hash(block.hash).unwrap()).collect() + } + + #[test] + fn inserts_and_reads_by_hash() { + let (_dir, store) = test_store(); + let hash = B256::random(); + let missing = NumHash::new(1, B256::random()); + let bal = Bytes::from_static(&[0xc1, 0x01]); + + store.insert(NumHash::new(1, hash), RawBal::from(bal.clone())).unwrap(); + + assert_eq!(read_many(&store, &[NumHash::new(1, hash), missing]), vec![Some(bal), None]); + } + + #[test] + fn hash_lookup_reads_persisted_bal_through_store() { + let (_dir, store) = test_store(); + let block = NumHash::new(1, B256::random()); + let bal = Bytes::from_static(&[0xc1, 0x01]); + + store.insert(block, RawBal::from(bal.clone())).unwrap(); + store.flush(&[block]).unwrap(); + + let store_with_empty_buffer = RocksDBBalStore::new(store.rocksdb_provider().clone()); + assert_eq!(store_with_empty_buffer.get_by_hash(block.hash).unwrap(), Some(bal)); + } + + #[test] + fn flush_prunes_buffer_retention() { + let (_dir, store) = test_store(); + let old = NumHash::new(1, B256::with_last_byte(1)); + let retained = + NumHash::new(DEFAULT_BAL_BUFFER_RETENTION_DISTANCE + 2, B256::with_last_byte(2)); + let old_bal = Bytes::from_static(&[0xc1, 0x01]); + let retained_bal = Bytes::from_static(&[0xc1, 0x02]); + + store.insert(old, RawBal::from(old_bal.clone())).unwrap(); + store.insert(retained, RawBal::from(retained_bal.clone())).unwrap(); + + assert_eq!( + store.get_by_hashes(&[old.hash, retained.hash]).unwrap(), + vec![Some(old_bal), Some(retained_bal.clone())] + ); + + store.flush(&[retained]).unwrap(); + + assert_eq!( + store.get_by_hashes(&[old.hash, retained.hash]).unwrap(), + vec![None, Some(retained_bal)] + ); + assert_eq!(disk_bal(&store, old), None); + } + + #[test] + fn flush_prunes_only_durable_cache_entries() { + let (_dir, store) = test_store(); + let old = NumHash::new(1, B256::with_last_byte(1)); + let tip = NumHash::new(DEFAULT_BAL_BUFFER_RETENTION_DISTANCE + 2, B256::with_last_byte(2)); + let old_bal = Bytes::from_static(&[0xc1, 0x01]); + + store.insert(old, RawBal::from(old_bal.clone())).unwrap(); + store.flush(&[old]).unwrap(); + store.flush(&[tip]).unwrap(); + + assert!(!store.buffer.read().entries.contains_key(&old.hash)); + assert_eq!(disk_bal(&store, old), Some(old_bal.clone())); + assert_eq!(store.get_by_hash(old.hash).unwrap(), Some(old_bal)); + } + + #[test] + fn configured_buffer_retention_distance_is_used() { + let dir = tempfile::tempdir().unwrap(); + let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); + let store = RocksDBBalStore::with_buffer_retention_distance(rocksdb, 64); + let old = NumHash::new(1, B256::with_last_byte(1)); + let tip = NumHash::new(34, B256::with_last_byte(2)); + + store.insert(old, RawBal::from(Bytes::from_static(&[0xc1, 0x01]))).unwrap(); + store.flush(&[old]).unwrap(); + store.flush(&[tip]).unwrap(); + + assert!(store.buffer.read().entries.contains_key(&old.hash)); + } + + #[test] + fn flush_writes_only_requested_pending_bals() { + let (_dir, store) = test_store(); + let block_1 = NumHash::new(1, B256::with_last_byte(1)); + let block_1_fork = NumHash::new(1, B256::with_last_byte(9)); + let block_2 = NumHash::new(2, B256::with_last_byte(2)); + let bal_1 = Bytes::from_static(&[0xc1, 0x01]); + let bal_1_fork = Bytes::from_static(&[0xc1, 0x09]); + let bal_2 = Bytes::from_static(&[0xc1, 0x02]); + + store.insert(block_1, RawBal::from(bal_1.clone())).unwrap(); + store.insert(block_1_fork, RawBal::from(bal_1_fork.clone())).unwrap(); + store.insert(block_2, RawBal::from(bal_2.clone())).unwrap(); + + store.flush(&[block_1]).unwrap(); + + assert_eq!(disk_bal(&store, block_1), Some(bal_1.clone())); + assert_eq!(disk_bal(&store, block_1_fork), None); + assert_eq!(disk_bal(&store, block_2), None); + assert_eq!( + store.get_by_hashes(&[block_1.hash, block_1_fork.hash, block_2.hash]).unwrap(), + vec![Some(bal_1.clone()), Some(bal_1_fork), Some(bal_2)] + ); + + let store_with_empty_buffer = RocksDBBalStore::new(store.rocksdb_provider().clone()); + assert_eq!( + store_with_empty_buffer + .get_by_hashes(&[block_1.hash, block_1_fork.hash, block_2.hash]) + .unwrap(), + vec![Some(bal_1), None, None] + ); + } + + #[test] + fn sparse_numbers_are_valid() { + let (_dir, store) = test_store(); + let hash_a = B256::with_last_byte(1); + let hash_b = B256::with_last_byte(2); + let bal_a = Bytes::from_static(&[0xc1, 0x01]); + let bal_b = Bytes::from_static(&[0xc1, 0x02]); + + let block_a = NumHash::new(2, hash_a); + let block_b = NumHash::new(200, hash_b); + + store.insert(block_a, RawBal::from(bal_a.clone())).unwrap(); + store.flush(&[block_a]).unwrap(); + store.insert(block_b, RawBal::from(bal_b.clone())).unwrap(); + store.flush(&[block_b]).unwrap(); + + assert_eq!(read_many(&store, &[block_a, block_b]), vec![Some(bal_a), Some(bal_b)]); + } + + #[test] + fn missing_and_empty_bal_are_distinct() { + let (_dir, store) = test_store(); + let empty_hash = B256::with_last_byte(1); + let missing_hash = B256::with_last_byte(2); + let empty_bal = Bytes::from_static(&[0xc0]); + + store.insert(NumHash::new(1, empty_hash), RawBal::from(empty_bal.clone())).unwrap(); + + assert_eq!( + read_many(&store, &[NumHash::new(1, empty_hash), NumHash::new(1, missing_hash)]), + vec![Some(empty_bal), None] + ); + } + + #[test] + fn prune_uses_eip_retention() { + let (_dir, store) = test_store(); + let old_hash = B256::with_last_byte(1); + let retained_hash = B256::with_last_byte(2); + let retained_bal = Bytes::from_static(&[0xc1, 0x02]); + let tip = BAL_RETENTION_PERIOD_SLOTS + 2; + + store + .insert(NumHash::new(1, old_hash), RawBal::from(Bytes::from_static(&[0xc1, 0x01]))) + .unwrap(); + store.insert(NumHash::new(2, retained_hash), RawBal::from(retained_bal.clone())).unwrap(); + store.flush(&[NumHash::new(1, old_hash), NumHash::new(2, retained_hash)]).unwrap(); + + assert_eq!(store.prune(tip).unwrap(), 1); + assert_eq!(disk_bal(&store, NumHash::new(1, old_hash)), None); + assert_eq!(disk_bal(&store, NumHash::new(2, retained_hash)), Some(retained_bal.clone())); + assert_eq!( + read_many(&store, &[NumHash::new(1, old_hash), NumHash::new(2, retained_hash)]), + vec![None, Some(retained_bal)] + ); + } + + #[test] + fn stored_payload_hash_is_not_reverified() { + let (_dir, store) = test_store(); + let block = NumHash::new(1, B256::with_last_byte(1)); + let key = StoredBlockAccessListKey::new(block.number, block.hash); + let value = StoredBlockAccessList::new_unchecked(B256::ZERO, Bytes::from_static(&[0xc0])); + + store.rocksdb_provider().put::(key, &value).unwrap(); + store + .rocksdb_provider() + .put::(block.hash, &block.number) + .unwrap(); + + assert_eq!(store.get_by_hash(block.hash).unwrap(), Some(Bytes::from_static(&[0xc0]))); + } + + #[test] + fn canonical_pending_entries_are_retried() { + let mut buffer = RocksDBBalStoreBuffer::default(); + let first = NumHash::new(1, B256::with_last_byte(1)); + let second = NumHash::new(2, B256::with_last_byte(2)); + buffer.insert(first, RawBal::from(Bytes::from_static(&[0xc1, 0x01]))); + buffer.mark_canonical(&[first]); + + assert_eq!(buffer.canonical_pending_entries().len(), 1); + + // A failed flush leaves the canonical entry pending for the next attempt. + buffer.insert(second, RawBal::from(Bytes::from_static(&[0xc1, 0x02]))); + buffer.mark_canonical(&[second]); + let retry = buffer.canonical_pending_entries(); + + assert_eq!( + retry.iter().map(|(key, _)| NumHash::new(key.number(), key.hash())).collect::>(), + vec![first, second] + ); + } + + #[tokio::test] + async fn insert_notifies_subscribers() { + let (_dir, store) = test_store(); + let mut stream = store.bal_stream(); + let block = NumHash::new(1, B256::with_last_byte(1)); + let bal = RawBal::from(Bytes::from_static(&[0xc0])); + + store.insert(block, bal.clone()).unwrap(); + + assert_eq!(stream.next().await.unwrap(), BalNotification::new(block, bal)); + } +} diff --git a/crates/storage/provider/src/lib.rs b/crates/storage/provider/src/lib.rs index 28c7c30058b..edb06faadb9 100644 --- a/crates/storage/provider/src/lib.rs +++ b/crates/storage/provider/src/lib.rs @@ -39,7 +39,7 @@ pub mod either_writer; pub use either_writer::*; mod bal; -pub use bal::{BalConfig, InMemoryBalStore}; +pub use bal::{BalConfig, InMemoryBalStore, RocksDBBalStore}; pub use reth_chain_state::{ CanonStateNotification, CanonStateNotificationSender, CanonStateNotificationStream, diff --git a/crates/storage/provider/src/providers/blockchain_provider.rs b/crates/storage/provider/src/providers/blockchain_provider.rs index 77800a2d591..3b861024584 100644 --- a/crates/storage/provider/src/providers/blockchain_provider.rs +++ b/crates/storage/provider/src/providers/blockchain_provider.rs @@ -241,7 +241,7 @@ impl NodePrimitivesProvider for BlockchainProvider { type Primitives = N::Primitives; } -impl BalProvider for BlockchainProvider { +impl BalProvider for BlockchainProvider { fn bal_store(&self) -> &BalStoreHandle { &self.bal_store } diff --git a/crates/storage/provider/src/providers/database/mod.rs b/crates/storage/provider/src/providers/database/mod.rs index fb2bb49b629..80c47356b37 100644 --- a/crates/storage/provider/src/providers/database/mod.rs +++ b/crates/storage/provider/src/providers/database/mod.rs @@ -604,7 +604,7 @@ impl NodePrimitivesProvider for ProviderFactory { type Primitives = N::Primitives; } -impl BalProvider for ProviderFactory { +impl BalProvider for ProviderFactory { fn bal_store(&self) -> &BalStoreHandle { &self.bal_store } diff --git a/crates/storage/provider/src/providers/rocksdb/metrics.rs b/crates/storage/provider/src/providers/rocksdb/metrics.rs index 3971ed978ba..c22edf65176 100644 --- a/crates/storage/provider/src/providers/rocksdb/metrics.rs +++ b/crates/storage/provider/src/providers/rocksdb/metrics.rs @@ -8,6 +8,8 @@ use strum::{EnumIter, IntoEnumIterator}; pub(super) const ROCKSDB_TABLES: &[&str] = &[ Tables::TransactionHashNumbers.name(), + Tables::BlockAccessLists.name(), + Tables::BlockAccessListBlockNumbers.name(), Tables::StoragesHistory.name(), Tables::AccountsHistory.name(), ]; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index 9a052121041..b7e3532dfb4 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -153,6 +153,14 @@ const DEFAULT_COMPRESS_BUF_CAPACITY: usize = 4096; /// The consistency check on startup heals any crash that occurs between auto-commits. const DEFAULT_AUTO_COMMIT_THRESHOLD: usize = 512 * 1024 * 1024; +/// Minimum BAL value size stored in `BlobDB` files. +/// +/// Smaller BALs stay inline. Larger payloads avoid regular LSM value compaction. +const DEFAULT_BAL_MIN_BLOB_SIZE: u64 = 4 * 1024; + +/// Target BAL blob file size. +const DEFAULT_BAL_BLOB_FILE_SIZE: u64 = 256 * 1024 * 1024; + /// Builder for [`RocksDBProvider`]. pub struct RocksDBBuilder { path: PathBuf, @@ -259,6 +267,16 @@ impl RocksDBBuilder { cf_options } + /// Creates column family options for block access list payloads. + fn block_access_lists_column_family_options(cache: &Cache) -> Options { + let mut cf_options = Self::default_column_family_options(cache); + cf_options.set_enable_blob_files(true); + cf_options.set_min_blob_size(DEFAULT_BAL_MIN_BLOB_SIZE); + cf_options.set_blob_file_size(DEFAULT_BAL_BLOB_FILE_SIZE); + cf_options.set_blob_compression_type(DBCompressionType::Lz4); + cf_options + } + /// Creates optimized column family options for `TransactionHashNumbers`. /// /// This table stores `B256 -> TxNumber` mappings where: @@ -297,10 +315,14 @@ impl RocksDBBuilder { /// - [`tables::TransactionHashNumbers`] - Transaction hash to number mapping /// - [`tables::AccountsHistory`] - Account history index /// - [`tables::StoragesHistory`] - Storage history index + /// - [`tables::BlockAccessLists`] - Block access list payloads + /// - [`tables::BlockAccessListBlockNumbers`] - Block access list hash index pub fn with_default_tables(self) -> Self { self.with_table::() .with_table::() .with_table::() + .with_table::() + .with_table::() } /// Enables metrics. @@ -352,6 +374,8 @@ impl RocksDBBuilder { .map(|name| { let cf_options = if name == tables::TransactionHashNumbers::NAME { Self::tx_hash_numbers_column_family_options(&self.block_cache) + } else if name == tables::BlockAccessLists::NAME { + Self::block_access_lists_column_family_options(&self.block_cache) } else { Self::default_column_family_options(&self.block_cache) }; @@ -1103,6 +1127,18 @@ impl RocksDBProvider { Ok(RocksDBRawIter { inner: iter }) } + /// Creates a raw key iterator positioned at `key`. + pub(crate) fn raw_key_iter_from( + &self, + key: T::Key, + ) -> ProviderResult> { + let cf = self.get_cf_handle::()?; + let encoded_key = key.encode(); + let mut iter = self.0.raw_iterator_cf(cf); + iter.seek(encoded_key.as_ref()); + Ok(RocksDBRawKeyIter { inner: iter }) + } + /// Returns all account history shards for the given address in ascending key order. /// /// This is used for unwind operations where we need to scan all shards for an address @@ -2730,6 +2766,39 @@ impl Iterator for RocksDBRawIter<'_> { } } +/// Raw key iterator over a `RocksDB` table (non-transactional). +pub(crate) struct RocksDBRawKeyIter<'db> { + inner: RocksDBRawIterEnum<'db>, +} + +impl fmt::Debug for RocksDBRawKeyIter<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("RocksDBRawKeyIter").finish_non_exhaustive() + } +} + +impl Iterator for RocksDBRawKeyIter<'_> { + type Item = ProviderResult>; + + fn next(&mut self) -> Option { + if !self.inner.valid() { + return self.inner.status().err().map(|e| { + Err(ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo { + message: e.to_string().into(), + code: -1, + }))) + }) + } + + let Some(key) = self.inner.key() else { + return Some(Err(ProviderError::Database(DatabaseError::Decode))) + }; + let key = Box::from(key); + self.inner.next(); + Some(Ok(key)) + } +} + /// Iterator over a `RocksDB` table within a transaction. /// /// Yields decoded `(Key, Value)` pairs. Sees uncommitted writes. @@ -2788,7 +2857,7 @@ const fn convert_log_level(level: LogLevel) -> rocksdb::LogLevel { mod tests { use super::*; use crate::providers::HistoryInfo; - use alloy_primitives::{Address, TxHash, B256}; + use alloy_primitives::{Address, Bytes, TxHash, B256}; use reth_db_api::{ models::{ sharded_key::{ShardedKey, NUM_OF_INDICES_IN_SHARD}, @@ -2822,6 +2891,41 @@ mod tests { let key = StorageShardedKey::new(Address::ZERO, B256::ZERO, 100); provider.put::(key.clone(), &value).unwrap(); assert!(provider.get::(key).unwrap().is_some()); + + let bal_key = + reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); + let bal_value = + reth_db_api::models::StoredBlockAccessList::new(Bytes::from_static(&[0xc0])); + provider.put::(bal_key, &bal_value).unwrap(); + assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); + provider + .put::(bal_key.hash(), &bal_key.number()) + .unwrap(); + assert_eq!( + provider.get::(bal_key.hash()).unwrap(), + Some(bal_key.number()) + ); + } + + #[test] + fn block_access_lists_store_large_payloads_in_blob_files() { + let temp_dir = TempDir::new().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let bal_key = + reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); + let bal_value = reth_db_api::models::StoredBlockAccessList::new(Bytes::from(vec![ + 0; + DEFAULT_BAL_MIN_BLOB_SIZE as usize + + 1 + ])); + + provider.put::(bal_key, &bal_value).unwrap(); + provider.flush(&[tables::BlockAccessLists::NAME]).unwrap(); + + let has_blob_file = std::fs::read_dir(temp_dir.path()).unwrap().any(|entry| { + entry.unwrap().path().extension().is_some_and(|extension| extension == "blob") + }); + assert!(has_blob_file); } #[derive(Debug)] diff --git a/crates/storage/storage-api/src/bal.rs b/crates/storage/storage-api/src/bal.rs index 26d485249b0..791b129cbc9 100644 --- a/crates/storage/storage-api/src/bal.rs +++ b/crates/storage/storage-api/src/bal.rs @@ -4,7 +4,6 @@ pub use alloy_eip7928::bal::RawBal; use alloy_eips::NumHash; use alloy_primitives::{BlockHash, BlockNumber, Bytes}; use reth_storage_errors::provider::ProviderResult; -use revm::database::state::bal::Bal as RevmBal; /// Notification emitted when a new BAL is inserted into the store. #[derive(Clone, Debug, PartialEq, Eq)] @@ -56,10 +55,10 @@ pub trait BalStore: Send + Sync + 'static { Ok(()) } - /// Flushes any pending BALs to the backing store. + /// Flushes pending BALs for the given canonical blocks to the backing store. /// /// In-memory implementations may treat this as a no-op. - fn flush(&self) -> ProviderResult<()> { + fn flush(&self, _blocks: &[NumHash]) -> ProviderResult<()> { Ok(()) } @@ -86,22 +85,6 @@ pub trait BalStore: Send + Sync + 'static { .map_err(Into::into) } - /// Fetches the BAL for the given block hash in revm representation. - fn revm_bal_by_hash( - &self, - block_hash: BlockHash, - ) -> ProviderResult>>> { - self.get_decoded_by_hash(block_hash)? - .map(|decoded| { - decoded.try_map(|bal| { - RevmBal::try_from(Vec::from(bal)) - .map(Arc::new) - .map_err(reth_storage_errors::provider::ProviderError::other) - }) - }) - .transpose() - } - /// Fetch BAL response entries for the given block hashes, stopping after the soft limit is /// exceeded. /// @@ -196,10 +179,10 @@ impl BalStoreHandle { self.inner.insert_many(entries) } - /// Flushes any pending BALs to the backing store. + /// Flushes pending BALs for the given canonical blocks to the backing store. #[inline] - pub fn flush(&self) -> ProviderResult<()> { - self.inner.flush() + pub fn flush(&self, blocks: &[NumHash]) -> ProviderResult<()> { + self.inner.flush(blocks) } /// Prunes expired BALs according to the store's retention policy and the given chain tip. @@ -226,15 +209,6 @@ impl BalStoreHandle { self.inner.get_decoded_by_hash(block_hash) } - /// Fetches the BAL for the given block hash in revm representation. - #[inline] - pub fn revm_bal_by_hash( - &self, - block_hash: BlockHash, - ) -> ProviderResult>>> { - self.inner.revm_bal_by_hash(block_hash) - } - /// Fetch BAL response entries for the given block hashes, stopping after the soft limit is /// exceeded. #[inline] @@ -282,6 +256,26 @@ impl core::fmt::Debug for BalStoreHandle { pub trait BalProvider { /// Returns the configured BAL store handle. fn bal_store(&self) -> &BalStoreHandle; + + /// Fetches the BAL for the given block hash. + fn get_bal_by_hash(&self, block_hash: BlockHash) -> ProviderResult> { + self.bal_store().get_by_hash(block_hash) + } + + /// Fetches BALs for the given block hashes. + fn get_bals_by_hashes(&self, block_hashes: &[BlockHash]) -> ProviderResult>> { + self.bal_store().get_by_hashes(block_hashes) + } + + /// Fetches BAL response entries for the given block hashes, stopping after the soft limit is + /// exceeded. + fn get_bals_by_hashes_with_limit( + &self, + block_hashes: &[BlockHash], + limit: GetBlockAccessListLimit, + ) -> ProviderResult>> { + self.bal_store().get_by_hashes_with_limit(block_hashes, limit) + } } /// No-op BAL store used as the default wiring target until a concrete implementation is injected. @@ -350,11 +344,29 @@ mod tests { assert_eq!(store.prune(10).unwrap(), 0); } + #[test] + fn noop_provider_returns_empty_results() { + let provider = crate::noop::NoopProvider::default(); + let hashes = [B256::random(), B256::random()]; + + assert_eq!(provider.get_bals_by_hashes(&hashes).unwrap(), vec![None, None]); + assert_eq!( + provider + .get_bals_by_hashes_with_limit( + &hashes, + GetBlockAccessListLimit::ResponseSizeSoftLimit(0), + ) + .unwrap(), + vec![None] + ); + assert!(provider.get_bal_by_hash(B256::random()).unwrap().is_none()); + } + #[test] fn noop_store_flush_is_noop() { let store = BalStoreHandle::default(); - store.flush().unwrap(); + store.flush(&[]).unwrap(); } #[test] @@ -375,11 +387,6 @@ mod tests { let decoded = store.get_decoded_by_hash(hash).unwrap().unwrap(); assert_eq!(decoded.as_raw(), &raw_bal); - - let revm_bal = store.revm_bal_by_hash(hash).unwrap().unwrap(); - - assert_eq!(revm_bal.as_raw(), &raw_bal); - assert!(revm_bal.as_bal().accounts.is_empty()); } #[test] From ddb640bb3f75436e6e3f758e5e5c73785727fab1 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Mon, 10 Aug 2026 22:37:54 +0200 Subject: [PATCH 116/148] ci: allow transitive bitmaps advisory (#26638) --- deny.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/deny.toml b/deny.toml index be45b22db44..133b02e82a1 100644 --- a/deny.toml +++ b/deny.toml @@ -12,6 +12,8 @@ ignore = [ "RUSTSEC-2023-0089", # https://rustsec.org/advisories/RUSTSEC-2026-0173 proc-macro-error2 is unmaintained "RUSTSEC-2026-0173", + # https://rustsec.org/advisories/RUSTSEC-2026-0247 bitmaps is unmaintained, transitive dep via imbl + "RUSTSEC-2026-0247", ] # This section is considered when running `cargo deny check bans`. From 1c6945c561d37a79857c97ee65c77338223ef639 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Mon, 10 Aug 2026 20:42:03 -0400 Subject: [PATCH 117/148] fix(network): do check the ECIES id in hello (#26639) --- crates/net/network/src/error.rs | 9 +- crates/net/network/src/session/mod.rs | 40 +++- crates/net/network/src/session/tests.rs | 290 ++++++++++++++++++++++++ 3 files changed, 333 insertions(+), 6 deletions(-) create mode 100644 crates/net/network/src/session/tests.rs diff --git a/crates/net/network/src/error.rs b/crates/net/network/src/error.rs index fee5326b047..7e3a84ba4eb 100644 --- a/crates/net/network/src/error.rs +++ b/crates/net/network/src/error.rs @@ -251,6 +251,9 @@ impl SessionError for PendingSessionHandshakeError { ECIESErrorImpl::Secp256k1(_) | ECIESErrorImpl::InvalidHandshake { .. } ), + // A peer that announces someone else's node id is broken or hostile, and its discovery + // record cannot be trusted to point at a usable peer. + Self::UnexpectedHandshakeIdentity(_) => true, Self::Timeout | Self::UnsupportedExtraCapability => false, } } @@ -270,7 +273,7 @@ impl SessionError for PendingSessionHandshakeError { ECIESErrorImpl::InvalidHandshake { .. } ), Self::Timeout => false, - Self::UnsupportedExtraCapability => true, + Self::UnsupportedExtraCapability | Self::UnexpectedHandshakeIdentity(_) => true, } } @@ -279,7 +282,9 @@ impl SessionError for PendingSessionHandshakeError { Self::Eth(eth) => eth.should_backoff(), Self::Ecies(_) => Some(BackoffKind::Low), Self::Timeout => Some(BackoffKind::Medium), - Self::UnsupportedExtraCapability => Some(BackoffKind::High), + Self::UnsupportedExtraCapability | Self::UnexpectedHandshakeIdentity(_) => { + Some(BackoffKind::High) + } } } } diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index 29d22f38543..db95a71d984 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -4,6 +4,8 @@ mod active; mod conn; mod counter; mod handle; +#[cfg(test)] +mod tests; mod types; pub use types::BlockRangeInfo; @@ -29,6 +31,7 @@ use reth_metrics::common::mpsc::MeteredPollSender; use reth_network_api::{PeerRequest, PeerRequestSender}; use reth_network_peers::PeerId; use reth_network_types::SessionsConfig; +use reth_primitives_traits::{GotExpected, GotExpectedBoxed}; use reth_tasks::Runtime; use rustc_hash::FxHashMap; use secp256k1::SecretKey; @@ -868,13 +871,21 @@ pub enum PendingSessionHandshakeError { /// Thrown when the remote lacks the required capability #[error("Mandatory extra capability unsupported")] UnsupportedExtraCapability, + /// Thrown when the node id in the remote's `Hello` message differs from the identity that the + /// ECIES handshake authenticated. + #[error("unexpected identity in hello message: {0}")] + UnexpectedHandshakeIdentity(GotExpectedBoxed), } impl PendingSessionHandshakeError { - /// Returns the [`DisconnectReason`] if the error is a disconnect message + /// Returns the [`DisconnectReason`] the session was disconnected with, either because the + /// remote sent one or because we sent one before failing the handshake. pub const fn as_disconnected(&self) -> Option { match self { Self::Eth(eth_err) => eth_err.as_disconnected(), + Self::UnexpectedHandshakeIdentity(_) => { + Some(DisconnectReason::UnexpectedHandshakeIdentity) + } _ => None, } } @@ -1104,6 +1115,8 @@ async fn authenticate_stream( // Add extra protocols to the hello message extra_handlers.retain(|handler| hello.try_add_protocol(handler.protocol()).is_ok()); + let authenticated_peer_id = stream.inner().remote_id(); + // conduct the p2p rlpx handshake and return the rlpx authenticated stream let (mut p2p_stream, their_hello) = match stream.handshake(hello).await { Ok(stream_res) => stream_res, @@ -1117,6 +1130,25 @@ async fn authenticate_stream( } }; + // The ECIES handshake proved possession of this key, so it is the only trustworthy identity of + // the connection. Everything the peer states in `Hello` is unauthenticated. + // + // Bind the session to the authenticated identity before anything else observes the peer. A + // peer that announces a different node id could otherwise act on behalf of that node, for + // example by making an extra protocol report reputation changes against it. + if their_hello.id != authenticated_peer_id { + let _ = p2p_stream.disconnect(DisconnectReason::UnexpectedHandshakeIdentity).await; + + return PendingSessionEvent::Disconnected { + remote_addr, + session_id, + direction, + error: Some(PendingSessionHandshakeError::UnexpectedHandshakeIdentity( + GotExpected { got: their_hello.id, expected: authenticated_peer_id }.into(), + )), + } + } + // if we have extra handlers, check if it must be supported by the remote if !extra_handlers.is_empty() { // ensure that no extra handlers that aren't supported are not mandatory @@ -1130,7 +1162,7 @@ async fn authenticate_stream( if handler.on_unsupported_by_peer( p2p_stream.shared_capabilities(), direction, - their_hello.id, + authenticated_peer_id, ) == OnNotSupported::Disconnect { return PendingSessionEvent::Disconnected { @@ -1213,7 +1245,7 @@ async fn authenticate_stream( // install additional handlers for handler in extra_handlers.into_iter() { let cap = handler.protocol().cap; - let remote_peer_id = their_hello.id; + let remote_peer_id = authenticated_peer_id; multiplex_stream .install_protocol(&cap, move |conn| { @@ -1247,7 +1279,7 @@ async fn authenticate_stream( session_id, remote_addr, local_addr, - peer_id: their_hello.id, + peer_id: authenticated_peer_id, capabilities: Arc::new(Capabilities::from(their_hello.capabilities)), status: Arc::new(their_status), conn, diff --git a/crates/net/network/src/session/tests.rs b/crates/net/network/src/session/tests.rs new file mode 100644 index 00000000000..d661f78ef80 --- /dev/null +++ b/crates/net/network/src/session/tests.rs @@ -0,0 +1,290 @@ +//! Tests for the session handshake. + +use super::*; +use crate::{ + error::SessionError, + protocol::{ConnectionHandler, DynConnectionHandler}, +}; +use alloy_primitives::bytes::BytesMut; +use futures::{stream::Pending, StreamExt}; +use reth_chainspec::MAINNET; +use reth_eth_wire::{ + capability::SharedCapabilities, handshake::EthHandshake, multiplex::ProtocolConnection, + protocol::Protocol, Capability, EthNetworkPrimitives, StatusBuilder, UnauthedEthStream, +}; +use reth_eth_wire_types::message::MAX_MESSAGE_SIZE; +use reth_ethereum_forks::EthereumHardfork; +use reth_network_peers::pk2id; +use secp256k1::SECP256K1; +use std::sync::Mutex; +use tokio::net::TcpListener; + +/// Extra `RLPx` protocol that records which identity the session hands to it. +#[derive(Clone, Debug, Default)] +struct ProtocolSpy { + seen: Arc>, +} + +/// The peers a [`ProtocolSpy`] was told about, per callback. +#[derive(Debug, Default)] +struct SeenPeers { + connected: Vec, + unsupported: Vec, +} + +impl ProtocolSpy { + /// The peers passed to [`ConnectionHandler::into_connection`]. + fn connected(&self) -> Vec { + self.seen.lock().unwrap().connected.clone() + } + + /// Whether the session named a peer to this protocol through either callback. + fn saw_any_peer(&self) -> bool { + let seen = self.seen.lock().unwrap(); + !seen.connected.is_empty() || !seen.unsupported.is_empty() + } +} + +/// The name sorts after `eth` so that `eth` keeps the first message id offset on both ends. +fn spy_protocol() -> Protocol { + Protocol::new(Capability::new_static("tst", 1), 1) +} + +impl ConnectionHandler for ProtocolSpy { + type Connection = Pending; + + fn protocol(&self) -> Protocol { + spy_protocol() + } + + fn on_unsupported_by_peer( + self, + _supported: &SharedCapabilities, + _direction: Direction, + peer_id: PeerId, + ) -> OnNotSupported { + self.seen.lock().unwrap().unsupported.push(peer_id); + OnNotSupported::KeepAlive + } + + fn into_connection( + self, + _direction: Direction, + peer_id: PeerId, + _conn: ProtocolConnection, + ) -> Self::Connection { + self.seen.lock().unwrap().connected.push(peer_id); + futures::stream::pending() + } +} + +fn new_peer() -> (SecretKey, PeerId) { + let (key, pk) = SECP256K1.generate_keypair(&mut rand_08::thread_rng()); + (key, pk2id(&pk)) +} + +fn status() -> UnifiedStatus { + StatusBuilder::default().build() +} + +fn fork_filter() -> ForkFilter { + MAINNET + .hardfork_fork_filter(EthereumHardfork::Frontier) + .expect("The Frontier fork filter should exist on mainnet") +} + +/// The hello a remote sends us, announcing `id` and the spy protocol. +fn remote_hello(id: PeerId) -> HelloMessageWithProtocols { + let mut hello = HelloMessageWithProtocols::builder(id).build(); + hello.try_add_protocol(spy_protocol()).unwrap(); + hello +} + +fn spy_handlers(spy: &ProtocolSpy) -> RlpxSubProtocolHandlers { + RlpxSubProtocolHandlers(vec![Box::new(spy.clone()) as Box]) +} + +/// Asserts the event reports an identity mismatch of `got` against `expected` and that the +/// peers manager will treat it as fatal. +fn assert_identity_mismatch( + event: PendingSessionEvent, + got: PeerId, + expected: PeerId, +) { + let PendingSessionEvent::Disconnected { error: Some(err), .. } = event else { + panic!("unexpected event {event:?}") + }; + let PendingSessionHandshakeError::UnexpectedHandshakeIdentity(ref mismatch) = err else { + panic!("unexpected error {err:?}") + }; + assert_eq!(mismatch.got, got); + assert_eq!(mismatch.expected, expected); + assert_eq!(err.as_disconnected(), Some(DisconnectReason::UnexpectedHandshakeIdentity)); + assert!(err.is_fatal_protocol_error()); +} + +/// An ECIES-authenticated peer that announces a different node id in its `Hello` must not be +/// able to make the session, or any extra protocol installed on it, act on that identity. +#[tokio::test(flavor = "multi_thread")] +async fn incoming_hello_with_spoofed_identity_is_rejected() { + let (local_key, local_id) = new_peer(); + let (attacker_key, attacker_id) = new_peer(); + let (_, victim_id) = new_peer(); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let spy = ProtocolSpy::default(); + let handlers = spy_handlers(&spy); + + let (events_tx, mut events_rx) = mpsc::channel(1); + let (_disconnect_tx, disconnect_rx) = oneshot::channel(); + + tokio::spawn(async move { + let (incoming, remote_addr) = listener.accept().await.unwrap(); + start_pending_incoming_session::( + Arc::new(EthHandshake::default()), + MAX_MESSAGE_SIZE, + disconnect_rx, + SessionId(0), + incoming, + events_tx, + remote_addr, + local_key, + HelloMessageWithProtocols::builder(local_id).build(), + status(), + fork_filter(), + handlers, + ) + .await + }); + + let attacker = tokio::spawn(async move { + let outgoing = TcpStream::connect(local_addr).await.unwrap(); + let ecies = ECIESStream::connect(outgoing, attacker_key, local_id).await.unwrap(); + let (mut p2p_stream, _) = + UnauthedP2PStream::new(ecies).handshake(remote_hello(victim_id)).await.unwrap(); + p2p_stream.next().await.unwrap().unwrap_err().as_disconnected() + }); + + assert_identity_mismatch(events_rx.recv().await.unwrap(), victim_id, attacker_id); + assert!(events_rx.recv().await.is_none(), "no session may be established"); + assert_eq!(attacker.await.unwrap(), Some(DisconnectReason::UnexpectedHandshakeIdentity)); + + assert!(!spy.saw_any_peer(), "the spoofed identity must not reach an extra protocol"); +} + +/// The same binding applies when we dial out: the remote proved possession of the key we +/// dialed, so its `Hello` cannot rename the session to a different peer. +#[tokio::test(flavor = "multi_thread")] +async fn outgoing_hello_with_spoofed_identity_is_rejected() { + let (local_key, local_id) = new_peer(); + let (remote_key, remote_id) = new_peer(); + let (_, victim_id) = new_peer(); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let remote_addr = listener.local_addr().unwrap(); + let spy = ProtocolSpy::default(); + let handlers = spy_handlers(&spy); + + let (events_tx, mut events_rx) = mpsc::channel(1); + let (_disconnect_tx, disconnect_rx) = oneshot::channel(); + + let remote = tokio::spawn(async move { + let (incoming, _) = listener.accept().await.unwrap(); + let ecies = ECIESStream::incoming(incoming, remote_key).await.unwrap(); + let (mut p2p_stream, _) = + UnauthedP2PStream::new(ecies).handshake(remote_hello(victim_id)).await.unwrap(); + p2p_stream.next().await.unwrap().unwrap_err().as_disconnected() + }); + + tokio::spawn(async move { + start_pending_outbound_session::( + Arc::new(EthHandshake::default()), + MAX_MESSAGE_SIZE, + disconnect_rx, + events_tx, + SessionId(0), + remote_addr, + remote_id, + local_key, + HelloMessageWithProtocols::builder(local_id).build(), + status(), + fork_filter(), + handlers, + ) + .await + }); + + assert_identity_mismatch(events_rx.recv().await.unwrap(), victim_id, remote_id); + assert!(events_rx.recv().await.is_none(), "no session may be established"); + assert_eq!(remote.await.unwrap(), Some(DisconnectReason::UnexpectedHandshakeIdentity)); + + assert!(!spy.saw_any_peer(), "the spoofed identity must not reach an extra protocol"); +} + +/// A remote whose `Hello` agrees with its ECIES identity gets a session, and the extra +/// protocol is handed that same identity. +#[tokio::test(flavor = "multi_thread")] +async fn matching_identity_establishes_session() { + let (local_key, local_id) = new_peer(); + let (remote_key, remote_id) = new_peer(); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let local_addr = listener.local_addr().unwrap(); + let spy = ProtocolSpy::default(); + let handlers = spy_handlers(&spy); + + let (events_tx, mut events_rx) = mpsc::channel(1); + let (_disconnect_tx, disconnect_rx) = oneshot::channel(); + + tokio::spawn(async move { + let (incoming, remote_addr) = listener.accept().await.unwrap(); + start_pending_incoming_session::( + Arc::new(EthHandshake::default()), + MAX_MESSAGE_SIZE, + disconnect_rx, + SessionId(0), + incoming, + events_tx, + remote_addr, + local_key, + HelloMessageWithProtocols::builder(local_id).build(), + status(), + fork_filter(), + handlers, + ) + .await + }); + + let (keep_alive_tx, keep_alive_rx) = oneshot::channel(); + let remote = tokio::spawn(async move { + let outgoing = TcpStream::connect(local_addr).await.unwrap(); + let ecies = ECIESStream::connect(outgoing, remote_key, local_id).await.unwrap(); + let (p2p_stream, _) = + UnauthedP2PStream::new(ecies).handshake(remote_hello(remote_id)).await.unwrap(); + + let mut status = status(); + status.set_eth_version(p2p_stream.shared_capabilities().eth_version().unwrap()); + let _eth_stream = UnauthedEthStream::new(p2p_stream) + .handshake::(status, fork_filter()) + .await + .unwrap(); + + let _ = keep_alive_rx.await; + }); + + let (peer_id, mut conn) = match events_rx.recv().await.unwrap() { + PendingSessionEvent::Established { peer_id, conn, .. } => (peer_id, conn), + event => panic!("unexpected event {event:?}"), + }; + assert_eq!(peer_id, remote_id); + assert_eq!(spy.connected(), vec![remote_id]); + + // From here on a real session owns and polls the connection. Without that the status we queued + // for the remote never reaches the socket and its handshake times out. + let session = tokio::spawn(async move { conn.next().await }); + + let _ = keep_alive_tx.send(()); + remote.await.unwrap(); + session.abort(); +} From 199387a02e0eeaab183cb602b47b391dcb616f8e Mon Sep 17 00:00:00 2001 From: Alexey Shekhirin Date: Tue, 11 Aug 2026 12:07:17 +0100 Subject: [PATCH 118/148] chore: expose blob cell availability on pooled transactions (#26642) --- crates/transaction-pool/src/traits.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/transaction-pool/src/traits.rs b/crates/transaction-pool/src/traits.rs index 59e9a062bcf..602d3ed02d6 100644 --- a/crates/transaction-pool/src/traits.rs +++ b/crates/transaction-pool/src/traits.rs @@ -1552,7 +1552,7 @@ pub struct EthPooledTransaction { /// Cached blob cell availability for this transaction. /// /// This is shared with the blob sidecar so that availability updates are reflected here. - blob_cell_availability: Option, + pub blob_cell_availability: Option, } impl EthPooledTransaction { From 62808bd060e4c7398e3fb6df93881950b1433b18 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Aug 2026 14:35:23 +0200 Subject: [PATCH 119/148] fix(txpool): allow senders with empty code hash (#26644) --- crates/transaction-pool/src/validate/eth.rs | 24 ++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/crates/transaction-pool/src/validate/eth.rs b/crates/transaction-pool/src/validate/eth.rs index eeb8f41dadc..79ab0afa54f 100644 --- a/crates/transaction-pool/src/validate/eth.rs +++ b/crates/transaction-pool/src/validate/eth.rs @@ -16,7 +16,7 @@ use crate::{ use alloy_consensus::{ constants::{ EIP1559_TX_TYPE_ID, EIP2930_TX_TYPE_ID, EIP4844_TX_TYPE_ID, EIP7702_TX_TYPE_ID, - LEGACY_TX_TYPE_ID, + KECCAK_EMPTY, LEGACY_TX_TYPE_ID, }, BlockHeader, }; @@ -732,7 +732,9 @@ where // // Any other case means that the account is not an EOA, and should not be able to send // transactions. - if let Some(code_hash) = &sender.bytecode_hash { + if let Some(code_hash) = &sender.bytecode_hash && + *code_hash != KECCAK_EMPTY + { let is_eip7702 = if self.fork_tracker.is_prague_activated() { match state.bytecode_by_hash(code_hash) { Ok(bytecode) => bytecode.unwrap_or_default().is_eip7702(), @@ -1552,7 +1554,7 @@ mod tests { eip2718::{Decodable2718, Encodable2718}, eip2930::{AccessList, AccessListItem}, }; - use alloy_primitives::{hex, Address, B256, U256}; + use alloy_primitives::{hex, Address, Bytes, B256, U256}; use reth_ethereum_primitives::PooledTransactionVariant; use reth_evm_ethereum::EthEvmConfig; use reth_primitives_traits::SignedTransaction; @@ -1685,6 +1687,22 @@ mod tests { assert!(tx.is_some()); } + #[test] + fn accepts_sender_with_empty_bytecode() { + let transaction = get_transaction(); + let provider = MockEthProvider::default().with_genesis_block(); + provider.add_account( + transaction.sender(), + ExtendedAccount::new(transaction.nonce(), U256::MAX).with_bytecode(Bytes::new()), + ); + let validator = EthTransactionValidatorBuilder::new(provider, test_evm_config()) + .build(InMemoryBlobStore::default()); + + let outcome = validator.validate_one(TransactionOrigin::External, transaction); + + assert!(outcome.is_valid()); + } + #[test] fn validates_configured_chain_id() { let provider = MockEthProvider::default().with_genesis_block(); From 905058c56e8fbbb8c40f633a051460ce6db5ec00 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Aug 2026 16:00:35 +0200 Subject: [PATCH 120/148] fix(storage): roll back RocksDB BAL storage (#26646) --- crates/cli/commands/src/common.rs | 16 ++++---- crates/engine/tree/src/persistence.rs | 37 ------------------- crates/node/builder/src/launch/common.rs | 19 +++++----- crates/storage/provider/src/bal/rocksdb.rs | 14 +++++-- .../provider/src/providers/rocksdb/metrics.rs | 2 - .../src/providers/rocksdb/provider.rs | 29 ++++++--------- 6 files changed, 38 insertions(+), 79 deletions(-) diff --git a/crates/cli/commands/src/common.rs b/crates/cli/commands/src/common.rs index 77cd552633f..2748ba5ec9c 100644 --- a/crates/cli/commands/src/common.rs +++ b/crates/cli/commands/src/common.rs @@ -27,7 +27,8 @@ use reth_provider::{ BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider, StaticFileProviderBuilder, }, - BalStoreHandle, ProviderFactory, RocksDBBalStore, StaticFileProviderFactory, StorageSettings, + BalConfig, BalStoreHandle, InMemoryBalStore, ProviderFactory, StaticFileProviderFactory, + StorageSettings, }; use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget}; use reth_static_file::StaticFileProducer; @@ -195,14 +196,11 @@ impl EnvironmentArgs { where C: ChainSpecParser, { - let bal_store = self - .db - .balstore_cache_size - .map(|distance| { - RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) - }) - .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); - let bal_store = BalStoreHandle::new(bal_store); + let balstore_cache_size = + self.db.balstore_cache_size.unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); + let bal_store = BalStoreHandle::new(InMemoryBalStore::new( + BalConfig::with_in_memory_retention_distance(balstore_cache_size), + )); let factory = ProviderFactory::>::new( db, self.chain.clone(), diff --git a/crates/engine/tree/src/persistence.rs b/crates/engine/tree/src/persistence.rs index 113e7a0f47a..feef9a6e368 100644 --- a/crates/engine/tree/src/persistence.rs +++ b/crates/engine/tree/src/persistence.rs @@ -566,43 +566,6 @@ mod tests { assert_eq!(result.last_state_trie_block, result.last_block); } - #[test] - fn test_save_blocks_flushes_bal_store() { - use reth_provider::{RocksDBBalStore, RocksDBProviderFactory}; - - reth_tracing::init_test_tracing(); - let provider = create_test_provider_factory(); - let bal_store = BalStoreHandle::new(RocksDBBalStore::new(provider.rocksdb_provider())); - let provider = provider.with_bal_store(bal_store); - init_genesis(&provider).unwrap(); - - let mut test_block_builder = TestBlockBuilder::eth(); - let executed = test_block_builder.get_executed_block_with_number(1, B256::random()); - let num_hash = executed.recovered_block().num_hash(); - let raw_bal = Bytes::from_static(&[0xc0]); - - provider.bal_store().insert(num_hash, RawBal::new(raw_bal.clone())).unwrap(); - - let (_finished_exex_height_tx, finished_exex_height_rx) = - tokio::sync::watch::channel(FinishedExExHeight::NoExExs); - let pruner = - Pruner::new_with_factory(provider.clone(), vec![], 5, 0, None, finished_exex_height_rx); - let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel(); - let handle = PersistenceHandle::::spawn_service( - provider.clone(), - pruner, - sync_metrics_tx, - ); - let (tx, rx) = crossbeam_channel::bounded(1); - - handle.save_blocks(full_save_input(vec![executed]), tx).unwrap(); - - let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out"); - assert_eq!(result.last_block, num_hash); - let persisted_store = RocksDBBalStore::new(provider.rocksdb_provider()); - assert_eq!(persisted_store.get_by_hash(num_hash.hash).unwrap(), Some(raw_bal)); - } - #[test] fn test_save_blocks_multiple_blocks() { reth_tracing::init_test_tracing(); diff --git a/crates/node/builder/src/launch/common.rs b/crates/node/builder/src/launch/common.rs index 43b6cc93bfc..498c65b3fab 100644 --- a/crates/node/builder/src/launch/common.rs +++ b/crates/node/builder/src/launch/common.rs @@ -71,10 +71,10 @@ use reth_node_metrics::{ }; use reth_provider::{ providers::{NodeTypesForProvider, ProviderNodeTypes, RocksDBProvider, StaticFileProvider}, - BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, DatabaseProviderFactory, - MetadataProvider, MetadataWriter, ProviderError, ProviderFactory, ProviderResult, - RocksDBBalStore, RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderBuilder, - StaticFileProviderFactory, StorageSettingsCache, + BalConfig, BalStoreHandle, BlockHashReader, BlockNumReader, DBProvider, + DatabaseProviderFactory, InMemoryBalStore, MetadataProvider, MetadataWriter, ProviderError, + ProviderFactory, ProviderResult, RocksDBProviderFactory, StageCheckpointReader, + StaticFileProviderBuilder, StaticFileProviderFactory, StorageSettingsCache, }; use reth_prune::{PruneMode, PruneModes, PrunerBuilder}; use reth_rpc_builder::config::RethRpcServerConfig; @@ -524,15 +524,14 @@ where .build()? }; - let bal_store = self + let balstore_cache_size = self .node_config() .db .balstore_cache_size - .map(|distance| { - RocksDBBalStore::with_buffer_retention_distance(rocksdb_provider.clone(), distance) - }) - .unwrap_or_else(|| RocksDBBalStore::new(rocksdb_provider.clone())); - let bal_store = BalStoreHandle::new(bal_store); + .unwrap_or(BalConfig::DEFAULT_IN_MEMORY_RETENTION_DISTANCE); + let bal_store = BalStoreHandle::new(InMemoryBalStore::new( + BalConfig::with_in_memory_retention_distance(balstore_cache_size), + )); let factory = ProviderFactory::new( self.right().clone(), self.chain_spec(), diff --git a/crates/storage/provider/src/bal/rocksdb.rs b/crates/storage/provider/src/bal/rocksdb.rs index d5886d1e8d9..99765824b5a 100644 --- a/crates/storage/provider/src/bal/rocksdb.rs +++ b/crates/storage/provider/src/bal/rocksdb.rs @@ -352,13 +352,21 @@ struct RocksDBBalEntry { #[cfg(test)] mod tests { use super::*; - use crate::providers::RocksDBBuilder; + use crate::providers::{RocksDBBuilder, RocksDBProvider}; use alloy_primitives::B256; use tokio_stream::StreamExt; + fn test_rocksdb(dir: &tempfile::TempDir) -> RocksDBProvider { + RocksDBBuilder::new(dir.path()) + .with_table::() + .with_table::() + .build() + .unwrap() + } + fn test_store() -> (tempfile::TempDir, RocksDBBalStore) { let dir = tempfile::tempdir().unwrap(); - let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); + let rocksdb = test_rocksdb(&dir); (dir, RocksDBBalStore::new(rocksdb)) } @@ -447,7 +455,7 @@ mod tests { #[test] fn configured_buffer_retention_distance_is_used() { let dir = tempfile::tempdir().unwrap(); - let rocksdb = RocksDBBuilder::new(dir.path()).with_default_tables().build().unwrap(); + let rocksdb = test_rocksdb(&dir); let store = RocksDBBalStore::with_buffer_retention_distance(rocksdb, 64); let old = NumHash::new(1, B256::with_last_byte(1)); let tip = NumHash::new(34, B256::with_last_byte(2)); diff --git a/crates/storage/provider/src/providers/rocksdb/metrics.rs b/crates/storage/provider/src/providers/rocksdb/metrics.rs index c22edf65176..3971ed978ba 100644 --- a/crates/storage/provider/src/providers/rocksdb/metrics.rs +++ b/crates/storage/provider/src/providers/rocksdb/metrics.rs @@ -8,8 +8,6 @@ use strum::{EnumIter, IntoEnumIterator}; pub(super) const ROCKSDB_TABLES: &[&str] = &[ Tables::TransactionHashNumbers.name(), - Tables::BlockAccessLists.name(), - Tables::BlockAccessListBlockNumbers.name(), Tables::StoragesHistory.name(), Tables::AccountsHistory.name(), ]; diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index b7e3532dfb4..e453537c255 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -315,14 +315,10 @@ impl RocksDBBuilder { /// - [`tables::TransactionHashNumbers`] - Transaction hash to number mapping /// - [`tables::AccountsHistory`] - Account history index /// - [`tables::StoragesHistory`] - Storage history index - /// - [`tables::BlockAccessLists`] - Block access list payloads - /// - [`tables::BlockAccessListBlockNumbers`] - Block access list hash index pub fn with_default_tables(self) -> Self { self.with_table::() .with_table::() .with_table::() - .with_table::() - .with_table::() } /// Enables metrics. @@ -2892,25 +2888,22 @@ mod tests { provider.put::(key.clone(), &value).unwrap(); assert!(provider.get::(key).unwrap().is_some()); - let bal_key = - reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); - let bal_value = - reth_db_api::models::StoredBlockAccessList::new(Bytes::from_static(&[0xc0])); - provider.put::(bal_key, &bal_value).unwrap(); - assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); - provider - .put::(bal_key.hash(), &bal_key.number()) - .unwrap(); - assert_eq!( - provider.get::(bal_key.hash()).unwrap(), - Some(bal_key.number()) - ); + drop(provider); + + let column_families = DB::list_cf(&Options::default(), temp_dir.path()).unwrap(); + assert!(!column_families.iter().any(|name| name == tables::BlockAccessLists::NAME)); + assert!(!column_families + .iter() + .any(|name| name == tables::BlockAccessListBlockNumbers::NAME)); } #[test] fn block_access_lists_store_large_payloads_in_blob_files() { let temp_dir = TempDir::new().unwrap(); - let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_table::() + .build() + .unwrap(); let bal_key = reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); let bal_value = reth_db_api::models::StoredBlockAccessList::new(Bytes::from(vec![ From c7338897a27fe1306eb6566cd163445555a9f7b5 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Aug 2026 16:38:39 +0200 Subject: [PATCH 121/148] fix(storage): tolerate unknown RocksDB column families (#26647) --- .../src/providers/rocksdb/provider.rs | 83 ++++++++++++++++++- 1 file changed, 81 insertions(+), 2 deletions(-) diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index e453537c255..d7198e7c74a 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -29,7 +29,7 @@ use rocksdb::{ BlockBasedOptions, Cache, ColumnFamilyDescriptor, CompactionPri, DBCompressionType, DBRawIteratorWithThreadMode, IteratorMode, OptimisticTransactionDB, OptimisticTransactionOptions, Options, SnapshotWithThreadMode, Transaction, - WriteBatchWithTransaction, WriteBufferManager, WriteOptions, DB, + WriteBatchWithTransaction, WriteBufferManager, WriteOptions, DB, DEFAULT_COLUMN_FAMILY_NAME, }; use std::{ collections::BTreeMap, @@ -364,7 +364,7 @@ impl RocksDBBuilder { let options = Self::default_options(self.log_level, &self.block_cache, self.enable_statistics); - let cf_descriptors: Vec = self + let mut cf_descriptors: Vec = self .column_families .iter() .map(|name| { @@ -379,6 +379,36 @@ impl RocksDBBuilder { }) .collect(); + // RocksDB requires every existing column family to be opened. Preserve column families + // unknown to this configuration so databases remain openable after a downgrade. + if RocksDBProvider::exists(&self.path) { + let existing_column_families = DB::list_cf(&options, &self.path).map_err(|e| { + ProviderError::Database(DatabaseError::Open(DatabaseErrorInfo { + message: e.to_string().into(), + code: -1, + })) + })?; + let unknown_column_families: Vec = existing_column_families + .into_iter() + .filter(|name| { + name != DEFAULT_COLUMN_FAMILY_NAME && !self.column_families.contains(name) + }) + .collect(); + if !unknown_column_families.is_empty() { + tracing::debug!( + target: "providers::rocksdb", + column_families = ?unknown_column_families, + "Preserving unknown column families" + ); + cf_descriptors.extend(unknown_column_families.into_iter().map(|name| { + ColumnFamilyDescriptor::new( + name, + Self::default_column_family_options(&self.block_cache), + ) + })); + } + } + let metrics = self.enable_metrics.then(RocksDBMetrics::default); if self.read_only { @@ -2931,6 +2961,55 @@ mod tests { type Value = Vec; } + #[test] + fn test_reopens_with_unknown_column_family() { + let temp_dir = TempDir::new().unwrap(); + let value = b"test_value".to_vec(); + + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_default_tables() + .with_table::() + .build() + .unwrap(); + provider.put::(42, &value).unwrap(); + drop(provider); + + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + assert_eq!(provider.get::(42).unwrap(), Some(value)); + } + + #[test] + fn test_reopens_blob_column_family_with_legacy_table_set() { + let temp_dir = TempDir::new().unwrap(); + let bal_key = + reth_db_api::models::StoredBlockAccessListKey::new(1, B256::with_last_byte(1)); + let bal_value = reth_db_api::models::StoredBlockAccessList::new(Bytes::from(vec![ + 0; + DEFAULT_BAL_MIN_BLOB_SIZE as usize + + 1 + ])); + + let provider = RocksDBBuilder::new(temp_dir.path()) + .with_default_tables() + .with_table::() + .with_table::() + .build() + .unwrap(); + provider.put::(bal_key, &bal_value).unwrap(); + provider + .put::(bal_key.hash(), &bal_key.number()) + .unwrap(); + provider.flush(&[tables::BlockAccessLists::NAME]).unwrap(); + drop(provider); + + let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap(); + assert_eq!(provider.get::(bal_key).unwrap(), Some(bal_value)); + assert_eq!( + provider.get::(bal_key.hash()).unwrap(), + Some(bal_key.number()) + ); + } + #[test] fn test_basic_operations() { let temp_dir = TempDir::new().unwrap(); From db9f6412d5baed3598b6bcb59b6c6a785de82c9f Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Tue, 11 Aug 2026 18:08:05 +0200 Subject: [PATCH 122/148] feat(network): allow configuring no-op client version (#26652) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> --- crates/net/network-api/src/noop.rs | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/crates/net/network-api/src/noop.rs b/crates/net/network-api/src/noop.rs index 200ebbd059c..f9f069dafe5 100644 --- a/crates/net/network-api/src/noop.rs +++ b/crates/net/network-api/src/noop.rs @@ -32,6 +32,7 @@ use tokio_stream::wrappers::UnboundedReceiverStream; #[non_exhaustive] pub struct NoopNetwork { chain_id: u64, + client_version: String, peers_handle: PeersHandle, cell_custody: CellCustody, _marker: PhantomData, @@ -44,6 +45,7 @@ impl NoopNetwork { Self { chain_id: 1, // mainnet + client_version: "reth-test".to_string(), peers_handle: PeersHandle::new(tx), cell_custody: CellCustody::default(), _marker: PhantomData, @@ -55,6 +57,12 @@ impl NoopNetwork { self.chain_id = chain_id; self } + + /// Configures the client version returned by [`NetworkInfo::network_status`]. + pub fn with_client_version(mut self, client_version: impl Into) -> Self { + self.client_version = client_version.into(); + self + } } impl Default for NoopNetwork { @@ -74,7 +82,7 @@ where async fn network_status(&self) -> Result { #[expect(deprecated)] Ok(NetworkStatus { - client_version: "reth-test".to_string(), + client_version: self.client_version.clone(), protocol_version: ProtocolVersion::V5 as u64, eth_protocol_info: EthProtocolInfo { network: 1, @@ -236,3 +244,18 @@ where &self.peers_handle } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn configures_client_version() { + let network = NoopNetwork::default().with_client_version("custom-client/v1.0.0"); + + assert_eq!( + futures::executor::block_on(network.network_status()).unwrap().client_version, + "custom-client/v1.0.0" + ); + } +} From f3e49cc6a6e3e60a71404315e9f1e9196df90bad Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 11 Aug 2026 19:06:57 +0200 Subject: [PATCH 123/148] fix(engine): adjust payload validation (#26651) --- crates/engine/tree/src/tree/payload_validator.rs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/crates/engine/tree/src/tree/payload_validator.rs b/crates/engine/tree/src/tree/payload_validator.rs index ea069a70fad..9c4dde82d67 100644 --- a/crates/engine/tree/src/tree/payload_validator.rs +++ b/crates/engine/tree/src/tree/payload_validator.rs @@ -168,10 +168,9 @@ use tracing::{debug, debug_span, error, info, instrument, trace, warn, Level, Sp pub use crate::tree::types::ValidationOutcome; -/// Multiplier over the parent's gas limit beyond which a block's claimed gas usage cannot be -/// legitimate. Gas limit can change by at most 1/1024 per block, so anything over this is rejected -/// without entering execution. -const MAX_EXPECTED_GAS_USAGE_MULTIPLIER: u64 = 2; +/// Multiplier over the parent's gas limit beyond which a payload must pass pre-execution +/// validation. +const MAX_EXPECTED_GAS_LIMIT_MULTIPLIER: u64 = 2; /// Worker name for deferred trie data preparation. const DEFERRED_TRIE_WORKER_NAME: &str = "deferred-trie"; @@ -530,9 +529,11 @@ where }; } - // If the gas usage is suspiciously high (multiple times higher than parent's gas limit), be - // cautious and block on pre-execution checks of the block. - if input.gas_used() > parent_block.gas_limit() * MAX_EXPECTED_GAS_USAGE_MULTIPLIER { + // If the gas limit is multiple times higher than the parent's, be cautious and block on + // pre-execution checks of the block. + if input.gas_limit() > + parent_block.gas_limit().saturating_mul(MAX_EXPECTED_GAS_LIMIT_MULTIPLIER) + { // Call `.get()` to await the pre-execution checks and exit early if they fail. if validated_block.get().is_err() { return Err(validated_block From 3a83ccc546bca3a90eb492fc0d115e6065aecb1a Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Tue, 11 Aug 2026 16:51:07 -0400 Subject: [PATCH 124/148] fix(eth-wire): reject out-of-range subprotocol message IDs (#26654) --- crates/net/eth-wire/src/capability.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/crates/net/eth-wire/src/capability.rs b/crates/net/eth-wire/src/capability.rs index e1dbd020ee3..b8ae5acaeb6 100644 --- a/crates/net/eth-wire/src/capability.rs +++ b/crates/net/eth-wire/src/capability.rs @@ -259,7 +259,8 @@ impl SharedCapabilities { cap = next } - Some(cap) + let end = u16::from(cap.message_id_offset()) + u16::from(cap.num_messages()); + (u16::from(offset) < end).then_some(cap) } /// Returns the shared capability for the given capability or an error if it's not compatible. @@ -466,6 +467,18 @@ mod tests { assert!(capabilities.supports_eth_v70()); } + #[test] + fn lookup_rejects_message_ids_past_last_capability() { + let cap = Capability::new_static("test", 1); + let shared = + SharedCapabilities::try_new(vec![Protocol::new(cap.clone(), 1)], vec![cap.clone()]) + .unwrap(); + let offset = shared.find(&cap).unwrap().message_id_offset(); + + assert_eq!(shared.find_by_offset(offset).unwrap().capability().as_ref(), &cap); + assert!(shared.find_by_offset(offset + 1).is_none()); + } + #[test] fn test_peer_capability_version_zero() { let cap = Capability::new_static("TestName", 0); From 189c0df32617afc488e0f091dbface1bd72cceb4 Mon Sep 17 00:00:00 2001 From: Emma Jamieson-Hoare Date: Wed, 12 Aug 2026 11:11:51 +0100 Subject: [PATCH 125/148] chore: bump reth version to 2.5.0 --- Cargo.lock | 228 +++++++++++++++++++-------------------- Cargo.toml | 2 +- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 116 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b33be037034..f84bacfb442 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3269,7 +3269,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.4.1" +version = "2.5.0" dependencies = [ "clap", "ef-tests", @@ -3277,7 +3277,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3730,7 +3730,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "eyre", @@ -3848,7 +3848,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "clap", @@ -7661,7 +7661,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7711,7 +7711,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7737,7 +7737,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7784,7 +7784,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7815,7 +7815,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7835,7 +7835,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-genesis", "clap", @@ -7847,7 +7847,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7937,7 +7937,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.4.1" +version = "2.5.0" dependencies = [ "reth-tasks", "tokio", @@ -7946,7 +7946,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8000,7 +8000,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "eyre", @@ -8018,7 +8018,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8031,7 +8031,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8045,7 +8045,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8069,7 +8069,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8105,7 +8105,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8133,7 +8133,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8164,7 +8164,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8180,7 +8180,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8206,7 +8206,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8231,7 +8231,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8259,7 +8259,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8297,7 +8297,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8353,7 +8353,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.4.1" +version = "2.5.0" dependencies = [ "aes", "alloy-primitives", @@ -8379,7 +8379,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8401,7 +8401,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8427,7 +8427,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8497,7 +8497,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8526,7 +8526,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8551,7 +8551,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "bytes", @@ -8569,7 +8569,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8600,7 +8600,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.4.1" +version = "2.5.0" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8610,7 +8610,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8648,7 +8648,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8676,7 +8676,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8716,7 +8716,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.4.1" +version = "2.5.0" dependencies = [ "clap", "eyre", @@ -8739,7 +8739,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8755,7 +8755,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8771,7 +8771,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.7", @@ -8784,7 +8784,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8813,7 +8813,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8834,7 +8834,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "rayon", @@ -8844,7 +8844,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8870,7 +8870,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8895,7 +8895,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "fixed-cache", @@ -8913,7 +8913,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-evm", "alloy-primitives", @@ -8925,7 +8925,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8946,7 +8946,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8991,7 +8991,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "eyre", @@ -9022,7 +9022,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9039,7 +9039,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "serde", "serde_json", @@ -9048,7 +9048,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9079,7 +9079,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.4.1" +version = "2.5.0" dependencies = [ "bytes", "futures", @@ -9101,7 +9101,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.4.1" +version = "2.5.0" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -9118,7 +9118,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.4.1" +version = "2.5.0" dependencies = [ "bindgen", "cc", @@ -9126,7 +9126,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.4.1" +version = "2.5.0" dependencies = [ "futures", "libc", @@ -9139,7 +9139,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "ipnet", @@ -9147,7 +9147,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.4.1" +version = "2.5.0" dependencies = [ "futures-util", "if-addrs", @@ -9161,7 +9161,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9230,7 +9230,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9254,7 +9254,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9278,7 +9278,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9295,7 +9295,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9308,7 +9308,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.4.1" +version = "2.5.0" dependencies = [ "anyhow", "bincode", @@ -9326,7 +9326,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9349,7 +9349,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9420,7 +9420,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9476,7 +9476,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9545,7 +9545,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9568,7 +9568,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9591,7 +9591,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.4.1" +version = "2.5.0" dependencies = [ "bytes", "eyre", @@ -9620,7 +9620,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9631,7 +9631,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9654,7 +9654,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9665,7 +9665,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9689,7 +9689,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9698,7 +9698,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9740,7 +9740,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9794,7 +9794,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9825,7 +9825,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -9845,7 +9845,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9861,7 +9861,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -9941,7 +9941,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-genesis", @@ -9971,7 +9971,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-network", @@ -10029,7 +10029,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-evm", @@ -10049,7 +10049,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -10069,7 +10069,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10105,7 +10105,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10151,7 +10151,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10203,7 +10203,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-rpc-types-engine", "http", @@ -10220,7 +10220,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10250,7 +10250,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10312,7 +10312,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10346,7 +10346,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "arbitrary", @@ -10362,7 +10362,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "assert_matches", @@ -10385,7 +10385,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "clap", @@ -10403,7 +10403,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10430,7 +10430,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10446,7 +10446,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10475,7 +10475,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10504,7 +10504,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.4.1" +version = "2.5.0" dependencies = [ "crossbeam-utils", "dashmap", @@ -10524,7 +10524,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10540,7 +10540,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.4.1" +version = "2.5.0" dependencies = [ "tokio", "tokio-stream", @@ -10549,7 +10549,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.4.1" +version = "2.5.0" dependencies = [ "clap", "eyre", @@ -10568,7 +10568,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.4.1" +version = "2.5.0" dependencies = [ "base64 0.22.1", "clap", @@ -10586,7 +10586,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10637,7 +10637,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10669,7 +10669,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10701,7 +10701,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10728,7 +10728,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10756,7 +10756,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.4.1" +version = "2.5.0" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index b23f6556a63..c82dd492175 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.4.1" +version = "2.5.0" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index bb8a8fba54b..29e1e41dae4 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.4.1', + text: 'v2.5.0', items: [ { text: 'Releases', From ab93eccf9c1538175841b9140c5bf7ebffc2b03b Mon Sep 17 00:00:00 2001 From: Francis Li Date: Thu, 20 Aug 2026 09:45:54 +0000 Subject: [PATCH 126/148] feat(payload): finalize pending build on resolve (#26759) Co-authored-by: Amp Co-authored-by: Arsenii Kulikov --- crates/payload/basic/src/lib.rs | 14 +++++--- crates/revm/src/cancelled.rs | 62 ++++++++++++++++++++++++++++++--- 2 files changed, 66 insertions(+), 10 deletions(-) diff --git a/crates/payload/basic/src/lib.rs b/crates/payload/basic/src/lib.rs index f9c18dfc00d..fb67012927f 100644 --- a/crates/payload/basic/src/lib.rs +++ b/crates/payload/basic/src/lib.rs @@ -416,7 +416,7 @@ where trace!(target: "payload_builder", id = %self.config.payload_id(), "spawn new payload build task"); let (tx, rx) = oneshot::channel(); let cancel = CancelOnDrop::default(); - let _cancel = cancel.clone(); + let pending_cancel = cancel.clone(); let guard = self.payload_task_guard.clone(); let payload_config = self.config.clone(); let best_payload = self.best_payload.payload().cloned(); @@ -446,7 +446,7 @@ where }); }); - self.pending_block = Some(PendingPayload { _cancel, payload: rx }); + self.pending_block = Some(PendingPayload { cancel: pending_cancel, payload: rx }); } } @@ -567,6 +567,10 @@ where let mut empty_payload = None; if best_payload.is_none() { + if let Some(pending) = maybe_better.as_ref() { + pending.cancel.request_finalization(); + } + debug!(target: "payload_builder", id=%self.config.payload_id(), "no best payload yet to resolve, building empty payload"); let args = BuildArguments { @@ -734,8 +738,8 @@ where /// A future that resolves to the result of the block building job. #[derive(Debug)] pub struct PendingPayload

{ - /// The marker to cancel the job on drop - _cancel: CancelOnDrop, + /// Cancels the job on drop and carries cooperative control signals. + cancel: CancelOnDrop, /// The channel to send the result to. payload: oneshot::Receiver, PayloadBuilderError>>, } @@ -746,7 +750,7 @@ impl

PendingPayload

{ cancel: CancelOnDrop, payload: oneshot::Receiver, PayloadBuilderError>>, ) -> Self { - Self { _cancel: cancel, payload } + Self { cancel, payload } } } diff --git a/crates/revm/src/cancelled.rs b/crates/revm/src/cancelled.rs index e535882355c..591cd3d376d 100644 --- a/crates/revm/src/cancelled.rs +++ b/crates/revm/src/cancelled.rs @@ -1,26 +1,53 @@ use alloc::sync::Arc; -use core::sync::atomic::AtomicBool; +use core::sync::atomic::{AtomicBool, AtomicU8, Ordering}; -/// A marker that can be used to cancel execution. +/// Payload building is still in progress. +const RUNNING: u8 = 0; +/// Payload building should stop accepting transactions and seal the accumulated work. +const FINALIZATION_REQUESTED: u8 = 1; +/// Payload building should stop and discard the accumulated work. +const CANCELLED: u8 = 2; + +/// Cancels execution on drop and supports cooperative finalization. /// /// If dropped, it will set the `cancelled` flag to true. /// /// This is most useful when a payload job needs to be cancelled. #[derive(Default, Clone, Debug)] -pub struct CancelOnDrop(Arc); +pub struct CancelOnDrop(Arc); // === impl CancelOnDrop === impl CancelOnDrop { + /// Returns true if the current work should be interrupted. + pub fn is_interrupted(&self) -> bool { + self.0.load(Ordering::Relaxed) != RUNNING + } + /// Returns true if the job was cancelled. pub fn is_cancelled(&self) -> bool { - self.0.load(core::sync::atomic::Ordering::Relaxed) + self.0.load(Ordering::Relaxed) == CANCELLED + } + + /// Requests that the current work be finalized without cancelling it. + pub fn request_finalization(&self) { + let _ = self.0.compare_exchange( + RUNNING, + FINALIZATION_REQUESTED, + Ordering::Relaxed, + Ordering::Relaxed, + ); + } + + /// Returns true if finalization was requested. + pub fn is_finalization_requested(&self) -> bool { + self.0.load(Ordering::Relaxed) == FINALIZATION_REQUESTED } } impl Drop for CancelOnDrop { fn drop(&mut self) { - self.0.store(true, core::sync::atomic::Ordering::Relaxed); + self.0.store(CANCELLED, Ordering::Relaxed); } } @@ -55,6 +82,7 @@ mod tests { #[test] fn test_default_cancelled() { let c = CancelOnDrop::default(); + assert!(!c.is_interrupted()); assert!(!c.is_cancelled()); } @@ -122,6 +150,7 @@ mod tests { drop(cancel); // The cloned instance should now see the cancelled flag as true + assert!(cloned_cancel.is_interrupted()); assert!(cloned_cancel.is_cancelled()); } @@ -140,8 +169,31 @@ mod tests { // Drop one clone - this should cancel all instances drop(clone1); + assert!(cancel.is_interrupted()); assert!(cancel.is_cancelled()); assert!(clone2.is_cancelled()); assert!(clone3.is_cancelled()); } + + #[test] + fn test_cancel_on_drop_finalization_request() { + let cancel = CancelOnDrop::default(); + let clone = cancel.clone(); + + cancel.request_finalization(); + + assert!(clone.is_interrupted()); + assert!(clone.is_finalization_requested()); + assert!(!clone.is_cancelled()); + + drop(cancel); + + assert!(clone.is_cancelled()); + assert!(!clone.is_finalization_requested()); + + clone.request_finalization(); + + assert!(clone.is_cancelled()); + assert!(!clone.is_finalization_requested()); + } } From b682925a8b491b26d663b3e373d66a18959cda3c Mon Sep 17 00:00:00 2001 From: Derek Cofausper <256792747+decofe@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:44:16 +0000 Subject: [PATCH 127/148] fix(provider): handle storage wipes in batched persistence (#26750) Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com> Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com> --- .../src/providers/database/provider.rs | 299 +++++++++++++----- 1 file changed, 215 insertions(+), 84 deletions(-) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index f29f84a5327..eb909288b6b 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -314,10 +314,10 @@ impl DatabaseProvider { &'a self, mut block_number: BlockNumber, ) -> ProviderResult> { - if block_number == self.best_block_number().unwrap_or_default() && - block_number == self.last_block_number().unwrap_or_default() + if block_number == self.best_block_number().unwrap_or_default() + && block_number == self.last_block_number().unwrap_or_default() { - return Ok(Box::new(LatestStateProviderRef::new(self))) + return Ok(Box::new(LatestStateProviderRef::new(self))); } // +1 as the changeset that we want is the one that was applied after this block. @@ -540,22 +540,22 @@ impl DatabaseProvider ProviderResult { let tip = self.last_block_number()?.max(last_block); Ok(StaticFileWriteCtx { - write_senders: EitherWriterDestination::senders(self).is_static_file() && - self.prune_modes.sender_recovery.is_none_or(|m| !m.is_full()), - write_receipts: save_mode.with_state() && - EitherWriter::receipts_destination(self).is_static_file(), - write_account_changesets: save_mode.with_state() && - EitherWriterDestination::account_changesets(self).is_static_file(), - write_storage_changesets: save_mode.with_state() && - EitherWriterDestination::storage_changesets(self).is_static_file(), + write_senders: EitherWriterDestination::senders(self).is_static_file() + && self.prune_modes.sender_recovery.is_none_or(|m| !m.is_full()), + write_receipts: save_mode.with_state() + && EitherWriter::receipts_destination(self).is_static_file(), + write_account_changesets: save_mode.with_state() + && EitherWriterDestination::account_changesets(self).is_static_file(), + write_storage_changesets: save_mode.with_state() + && EitherWriterDestination::storage_changesets(self).is_static_file(), tip, receipts_prune_mode: self.prune_modes.receipts, // Receipts are prunable if no receipts exist in SF yet and within pruning distance receipts_prunable: self .static_file_provider .get_highest_static_file_tx(StaticFileSegment::Receipts) - .is_none() && - PruneMode::Distance(self.minimum_pruning_distance) + .is_none() + && PruneMode::Distance(self.minimum_pruning_distance) .should_prune(first_block, tip), }) } @@ -594,7 +594,7 @@ impl DatabaseProvider DatabaseProvider DatabaseProvider>(); - let mask = state_trie_masking_blocks - .iter() - .map(|block| block.trie_data.get().sorted.trie_updates.as_ref()) - .collect::>(); - let merged_trie = TrieUpdatesSorted::disjointed_merge_batch(&batch, &mask); + let batch = ExecutedBlock::trie_updates_refs(state_trie_blocks); + let mask = ExecutedBlock::trie_updates_refs(state_trie_masking_blocks); + // Sparse-trie streaming emits node updates, while the serial state-root path, + // including fallback, can emit a whole storage-trie wipe. `disjointed_merge_batch` + // rejects wipes, so use the ordered merge. Persisting updates shadowed by the + // masking suffix is redundant but safe: the in-memory overlay still takes + // precedence. + // + // With a revm release containing bluealloy/revm#3863, post-Cancun selfdestructs + // will no longer result in `storage.is_deleted` in serial trie updates. The flag + // remains valid `save_blocks` input when processing pre-Cancun historical data. + let contains_storage_wipe = batch.iter().chain(&mask).any(|updates| { + updates.storage_tries_ref().values().any(|storage| storage.is_deleted) + }); + let merged_trie = if contains_storage_wipe { + TrieUpdatesSorted::merge_batch( + state_trie_blocks.iter().rev().map(|block| block.trie_updates()), + ) + } else { + Arc::new(TrieUpdatesSorted::disjointed_merge_batch(&batch, &mask)) + }; if !merged_trie.is_empty() { self.write_trie_updates_sorted(&merged_trie)?; } @@ -806,8 +818,8 @@ impl DatabaseProvider DatabaseProvider>, first_tx_num: TxNumber, ) -> ProviderResult { - if self.prune_modes.sender_recovery.is_none_or(|m| !m.is_full()) && - EitherWriterDestination::senders(self).is_database() + if self.prune_modes.sender_recovery.is_none_or(|m| !m.is_full()) + && EitherWriterDestination::senders(self).is_database() { let start = Instant::now(); let tx_nums_iter = std::iter::successors(Some(first_tx_num), |n| Some(n + 1)); @@ -1084,7 +1096,7 @@ where while let Some((sharded_key, list)) = item { // If the shard does not belong to the key, break. if !shard_belongs_to_key(&sharded_key) { - break + break; } // Always delete the current shard from the database first @@ -1099,18 +1111,18 @@ where // Keep it deleted (don't return anything for reinsertion) if first >= block_number { item = cursor.prev()?; - continue + continue; } // Case 2: This is a boundary shard (spans across the unwinding point) // The shard contains some blocks below and some at/above the unwinding point else if block_number <= sharded_key.as_ref().highest_block_number { // Return only the block numbers that are below the unwinding point // These will be reinserted to preserve the historical data - return Ok(list.iter().take_while(|i| *i < block_number).collect::>()) + return Ok(list.iter().take_while(|i| *i < block_number).collect::>()); } // Case 3: Entire shard is below the unwinding point // Return all block numbers for reinsertion (preserve entire shard) - return Ok(list.iter().collect::>()) + return Ok(list.iter().collect::>()); } // No shards found or all processed @@ -1189,7 +1201,7 @@ impl DatabaseProvider { let Some(block_number) = self.convert_hash_or_number(id)? else { return Ok(None) }; let earliest_available = self.static_file_provider.earliest_history_height(); if block_number < earliest_available { - return Err(ProviderError::BlockExpired { requested: block_number, earliest_available }) + return Err(ProviderError::BlockExpired { requested: block_number, earliest_available }); } let Some(header) = header_by_number(block_number)? else { return Ok(None) }; @@ -1259,7 +1271,7 @@ impl DatabaseProvider { F: FnMut(H, BodyTy, Range) -> ProviderResult, { if range.is_empty() { - return Ok(Vec::new()) + return Ok(Vec::new()); } let len = range.end().saturating_sub(*range.start()) as usize + 1; @@ -1594,8 +1606,8 @@ impl AccountExtReader for DatabaseProvider HeaderSyncGapProvider } Ordering::Less => { // There's either missing or corrupted files. - return Err(ProviderError::HeaderNotFound(next_static_file_block_num.into())) + return Err(ProviderError::HeaderNotFound(next_static_file_block_num.into())); } Ordering::Equal => {} } @@ -1928,7 +1940,7 @@ impl BlockReader for DatabaseProvid if let Some(number) = self.convert_hash_or_number(id)? { let earliest_available = self.static_file_provider.earliest_history_height(); if number < earliest_available { - return Err(ProviderError::BlockExpired { requested: number, earliest_available }) + return Err(ProviderError::BlockExpired { requested: number, earliest_available }); } let Some(header) = self.header_by_number(number)? else { return Ok(None) }; @@ -1938,7 +1950,7 @@ impl BlockReader for DatabaseProvid // If they exist but are not indexed, we don't have enough // information to return the block anyways, so we return `None`. let Some(transactions) = self.transactions_by_block(number.into())? else { - return Ok(None) + return Ok(None); }; let body = self @@ -1948,7 +1960,7 @@ impl BlockReader for DatabaseProvid .pop() .ok_or(ProviderError::InvalidStorageOutput)?; - return Ok(Some(Self::Block::new(header, body))) + return Ok(Some(Self::Block::new(header, body))); } Ok(None) @@ -2108,10 +2120,10 @@ impl TransactionsProvider for Datab &self, tx_hash: TxHash, ) -> ProviderResult> { - if let Some(transaction_id) = self.transaction_id(tx_hash)? && - let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? && - let Some(block_number) = self.block_by_transaction_id(transaction_id)? && - let Some(sealed_header) = self.sealed_header(block_number)? + if let Some(transaction_id) = self.transaction_id(tx_hash)? + && let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? + && let Some(block_number) = self.block_by_transaction_id(transaction_id)? + && let Some(sealed_header) = self.sealed_header(block_number)? { let (header, block_hash) = sealed_header.split(); if let Some(block_body) = self.block_body_indices(block_number)? { @@ -2131,7 +2143,7 @@ impl TransactionsProvider for Datab timestamp: header.timestamp(), }; - return Ok(Some((transaction, meta))) + return Ok(Some((transaction, meta))); } } @@ -2142,15 +2154,15 @@ impl TransactionsProvider for Datab &self, id: BlockHashOrNumber, ) -> ProviderResult>> { - if let Some(block_number) = self.convert_hash_or_number(id)? && - let Some(body) = self.block_body_indices(block_number)? + if let Some(block_number) = self.convert_hash_or_number(id)? + && let Some(body) = self.block_body_indices(block_number)? { let tx_range = body.tx_num_range(); return if tx_range.is_empty() { Ok(Some(Vec::new())) } else { self.transactions_by_tx_range(tx_range).map(Some) - } + }; } Ok(None) } @@ -2225,8 +2237,8 @@ impl ReceiptProvider for DatabasePr &self, block: BlockHashOrNumber, ) -> ProviderResult>> { - if let Some(number) = self.convert_hash_or_number(block)? && - let Some(body) = self.block_body_indices(number)? + if let Some(number) = self.convert_hash_or_number(block)? + && let Some(body) = self.block_body_indices(number)? { let tx_range = body.tx_num_range(); return if tx_range.is_empty() { @@ -2235,11 +2247,11 @@ impl ReceiptProvider for DatabasePr let receipts = self.receipts_by_tx_range(tx_range)?; if receipts.len() != body.tx_count as usize { - return Ok(None) + return Ok(None); } Ok(Some(receipts)) - } + }; } Ok(None) } @@ -2556,10 +2568,10 @@ impl StateWriter ) -> ProviderResult<()> { let execution_outcome = execution_outcome.into(); - if self.cached_storage_settings().use_hashed_state() && - !config.write_receipts && - !config.write_account_changesets && - !config.write_storage_changesets + if self.cached_storage_settings().use_hashed_state() + && !config.write_receipts + && !config.write_account_changesets + && !config.write_storage_changesets { // In storage v2 with all outputs directed to static files, plain state and changesets // are written elsewhere. Only bytecodes need MDBX writes, so skip the expensive @@ -2614,11 +2626,12 @@ impl StateWriter // Database, OR if receipts_in_static_files is enabled but no receipts exist in static // files yet. Once receipts exist in static files, we must continue writing to maintain // continuity and have no gaps. - let prunable_receipts = (EitherWriter::receipts_destination(self).is_database() || - self.static_file_provider() + let prunable_receipts = (EitherWriter::receipts_destination(self).is_database() + || self + .static_file_provider() .get_highest_static_file_tx(StaticFileSegment::Receipts) - .is_none()) && - PruneMode::Distance(self.minimum_pruning_distance).should_prune(first_block, tip); + .is_none()) + && PruneMode::Distance(self.minimum_pruning_distance).should_prune(first_block, tip); // Prepare set of addresses which logs should not be pruned. let mut allowed_addresses: AddressSet = AddressSet::default(); @@ -2635,12 +2648,13 @@ impl StateWriter receipts_writer.increment_block(block_number)?; // Skip writing receipts if pruning configuration requires us to. - if prunable_receipts && - self.prune_modes + if prunable_receipts + && self + .prune_modes .receipts .is_some_and(|mode| mode.should_prune(block_number, tip)) { - continue + continue; } // If there are new addresses to retain after this block number, track them @@ -2652,11 +2666,11 @@ impl StateWriter let receipt_idx = first_tx_index + idx as u64; // Skip writing receipt if log filter is active and it does not have any logs to // retain - if prunable_receipts && - has_contract_log_filter && - !receipt.logs().iter().any(|log| allowed_addresses.contains(&log.address)) + if prunable_receipts + && has_contract_log_filter + && !receipt.logs().iter().any(|log| allowed_addresses.contains(&log.address)) { - continue + continue; } receipts_writer.append_receipt(receipt_idx, receipt)?; @@ -2787,8 +2801,8 @@ impl StateWriter for entry in storage { tracing::trace!(?address, ?entry.key, "Updating plain state storage"); if let Some(db_entry) = - storages_cursor.seek_by_key_subkey(address, entry.key)? && - db_entry.key == entry.key + storages_cursor.seek_by_key_subkey(address, entry.key)? + && db_entry.key == entry.key { storages_cursor.delete_current()?; } @@ -2834,8 +2848,8 @@ impl StateWriter let entry = StorageEntry { key: *hashed_slot, value: *value }; if let Some(db_entry) = - hashed_storage_cursor.seek_by_key_subkey(*hashed_address, entry.key)? && - db_entry.key == entry.key + hashed_storage_cursor.seek_by_key_subkey(*hashed_address, entry.key)? + && db_entry.key == entry.key { hashed_storage_cursor.delete_current()?; } @@ -3018,7 +3032,7 @@ impl StateWriter let range = block + 1..=self.last_block_number()?; if range.is_empty() { - return Ok(ExecutionOutcome::default()) + return Ok(ExecutionOutcome::default()); } let start_block_number = *range.start(); @@ -3034,8 +3048,8 @@ impl StateWriter let storage_range = BlockNumberAddress::range(range.clone()); let storage_changeset = if let Some(highest_block) = self .static_file_provider - .get_highest_static_file_block(StaticFileSegment::StorageChangeSets) && - self.cached_storage_settings().storage_v2 + .get_highest_static_file_block(StaticFileSegment::StorageChangeSets) + && self.cached_storage_settings().storage_v2 { let changesets = self.storage_changesets_range(block + 1..=highest_block)?; let mut changeset_writer = @@ -3050,8 +3064,8 @@ impl StateWriter let highest_changeset_block = self .static_file_provider .get_highest_static_file_block(StaticFileSegment::AccountChangeSets); - let account_changeset = if let Some(highest_block) = highest_changeset_block && - self.cached_storage_settings().storage_v2 + let account_changeset = if let Some(highest_block) = highest_changeset_block + && self.cached_storage_settings().storage_v2 { // TODO: add a `take` method that removes and returns the items instead of doing this let changesets = self.account_changesets_range(block + 1..highest_block + 1)?; @@ -3258,7 +3272,7 @@ impl TrieWriter for DatabaseProvider #[instrument(level = "debug", target = "providers::db", skip_all)] fn write_trie_updates_sorted(&self, trie_updates: &TrieUpdatesSorted) -> ProviderResult { if trie_updates.is_empty() { - return Ok(0) + return Ok(0); } // Track the number of inserted entries. @@ -3516,8 +3530,8 @@ impl HistoryWriter for DatabaseProvi StorageShardedKey::last(address, storage_key), rem_index, |storage_sharded_key| { - storage_sharded_key.address == address && - storage_sharded_key.sharded_key.key == storage_key + storage_sharded_key.address == address + && storage_sharded_key.sharded_key.key == storage_key }, )?; @@ -3817,7 +3831,7 @@ impl BlockWriter ) -> ProviderResult<()> { if blocks.is_empty() { debug!(target: "providers::db", "Attempted to append empty block range"); - return Ok(()) + return Ok(()); } // Blocks are not empty, so no need to handle the case of `blocks.first()` being @@ -4858,6 +4872,123 @@ mod tests { assert_eq!(masked_entries[0].1.nibbles.0, masked_storage_node); } + #[cfg(feature = "partial-persistence")] + #[test] + fn test_save_blocks_merges_storage_wipe_in_multi_block_batch() { + use reth_trie::{ + updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted}, + BranchNodeCompact, HashedPostStateSorted, + }; + + let factory = create_test_provider_factory(); + factory.set_storage_settings_cache(StorageSettings::v2()); + + let mut test_block_builder = TestBlockBuilder::eth().with_state(); + let genesis = test_block_builder.get_executed_blocks(0..1).next().unwrap(); + let mut blocks: Vec<_> = test_block_builder.get_executed_blocks(1..4).collect(); + let address = B256::with_last_byte(1); + let storage_path = Nibbles::from_nibbles([0x1, 0x2]); + + let provider_rw = factory.provider_rw().unwrap(); + save_genesis(&provider_rw, &genesis).unwrap(); + provider_rw + .write_trie_updates_sorted(&TrieUpdatesSorted::new( + vec![], + B256Map::from_iter([( + address, + StorageTrieUpdatesSorted { + is_deleted: false, + storage_nodes: vec![(storage_path, Some(BranchNodeCompact::default()))], + }, + )]), + )) + .unwrap(); + provider_rw.commit().unwrap(); + + let wipe = TrieUpdatesSorted::new( + vec![], + B256Map::from_iter([( + address, + StorageTrieUpdatesSorted { is_deleted: true, storage_nodes: vec![] }, + )]), + ); + let wipe_block = &blocks[2]; + blocks[2] = ExecutedBlock::new( + Arc::clone(&wipe_block.recovered_block), + Arc::clone(&wipe_block.execution_output), + ComputedTrieData::new(Arc::new(HashedPostStateSorted::default()), Arc::new(wipe)), + ); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new(blocks, 0, 0, 3, 3); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let provider = factory.provider().unwrap(); + let checkpoint = provider.get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); + assert_eq!(checkpoint.block_number, 3); + let storage_entries = provider + .tx_ref() + .cursor_dup_read::() + .unwrap() + .walk_dup(Some(address), None) + .unwrap() + .collect::, _>>() + .unwrap(); + assert!(storage_entries.is_empty()); + } + + #[cfg(feature = "partial-persistence")] + #[test] + fn test_save_blocks_batches_transient_storage_wipe() { + use alloy_primitives::map::B256Set; + use reth_trie::{updates::TrieUpdates, HashBuilder, HashedPostStateSorted}; + + let factory = create_test_provider_factory(); + factory.set_storage_settings_cache(StorageSettings::v2()); + + let mut test_block_builder = TestBlockBuilder::eth().with_state(); + let genesis = test_block_builder.get_executed_blocks(0..1).next().unwrap(); + let blocks: Vec<_> = test_block_builder.get_executed_blocks(1..4).collect(); + + let provider_rw = factory.provider_rw().unwrap(); + save_genesis(&provider_rw, &genesis).unwrap(); + provider_rw.commit().unwrap(); + + // A contract created and self-destructed in the same transaction leaves an empty whole- + // storage wipe in the trie updates, even though the account never reaches persisted state. + let ephemeral_account = B256::with_last_byte(0x57); + let hashed_state = HashedPostStateSorted::default(); + let mut trie_updates = TrieUpdates::default(); + trie_updates.finalize( + HashBuilder::default(), + Default::default(), + B256Set::from_iter([ephemeral_account]), + ); + let trie_updates = trie_updates.into_sorted(); + assert!(trie_updates.storage_tries_ref()[&ephemeral_account].is_deleted); + let selfdestruct_block = ExecutedBlock::new( + Arc::clone(&blocks[2].recovered_block), + Arc::clone(&blocks[2].execution_output), + ComputedTrieData::new(Arc::new(hashed_state), Arc::new(trie_updates)), + ); + + let provider_rw = factory.provider_rw().unwrap(); + let input = SaveBlocksInput::new( + vec![blocks[0].clone(), blocks[1].clone(), selfdestruct_block], + 0, + 0, + 3, + 3, + ); + provider_rw.save_blocks(&input).unwrap(); + provider_rw.commit().unwrap(); + + let checkpoint = + factory.provider().unwrap().get_stage_checkpoint(StageId::Finish).unwrap().unwrap(); + assert_eq!(checkpoint.block_number, 3); + } + #[cfg(feature = "partial-persistence")] #[test] fn test_save_blocks_partial_cycles_do_not_duplicate_static_file_writes() { @@ -5540,8 +5671,8 @@ mod tests { ..Default::default() }; - let storage: HashMap> = (1..= - slots_per_account as u64) + let storage: HashMap> = (1 + ..=slots_per_account as u64) .map(|s| { ( U256::from(s + acct_idx as u64 * 100), From b7a801df625d96a87de1b227504ee41af6a8ffa5 Mon Sep 17 00:00:00 2001 From: Emma Jamieson-Hoare Date: Thu, 20 Aug 2026 11:40:02 +0100 Subject: [PATCH 128/148] chore: bump versions --- Cargo.lock | 228 +++++++++++++++++++-------------------- Cargo.toml | 2 +- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 116 insertions(+), 116 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f84bacfb442..04c928a3f41 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3269,7 +3269,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.5.0" +version = "2.5.1" dependencies = [ "clap", "ef-tests", @@ -3277,7 +3277,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3730,7 +3730,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "eyre", @@ -3848,7 +3848,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "clap", @@ -7661,7 +7661,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7711,7 +7711,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7737,7 +7737,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7784,7 +7784,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7815,7 +7815,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7835,7 +7835,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-genesis", "clap", @@ -7847,7 +7847,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7937,7 +7937,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.5.0" +version = "2.5.1" dependencies = [ "reth-tasks", "tokio", @@ -7946,7 +7946,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8000,7 +8000,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "eyre", @@ -8018,7 +8018,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8031,7 +8031,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8045,7 +8045,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8069,7 +8069,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8105,7 +8105,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8133,7 +8133,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8164,7 +8164,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8180,7 +8180,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8206,7 +8206,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8231,7 +8231,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8259,7 +8259,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8297,7 +8297,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8353,7 +8353,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.5.0" +version = "2.5.1" dependencies = [ "aes", "alloy-primitives", @@ -8379,7 +8379,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8401,7 +8401,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8427,7 +8427,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8497,7 +8497,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8526,7 +8526,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8551,7 +8551,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "bytes", @@ -8569,7 +8569,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8600,7 +8600,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.5.0" +version = "2.5.1" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8610,7 +8610,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8648,7 +8648,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8676,7 +8676,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8716,7 +8716,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.5.0" +version = "2.5.1" dependencies = [ "clap", "eyre", @@ -8739,7 +8739,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8755,7 +8755,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8771,7 +8771,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.7", @@ -8784,7 +8784,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8813,7 +8813,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8834,7 +8834,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "rayon", @@ -8844,7 +8844,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8870,7 +8870,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8895,7 +8895,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "fixed-cache", @@ -8913,7 +8913,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-evm", "alloy-primitives", @@ -8925,7 +8925,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8946,7 +8946,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8991,7 +8991,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "eyre", @@ -9022,7 +9022,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9039,7 +9039,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "serde", "serde_json", @@ -9048,7 +9048,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9079,7 +9079,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.5.0" +version = "2.5.1" dependencies = [ "bytes", "futures", @@ -9101,7 +9101,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.5.0" +version = "2.5.1" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -9118,7 +9118,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.5.0" +version = "2.5.1" dependencies = [ "bindgen", "cc", @@ -9126,7 +9126,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.5.0" +version = "2.5.1" dependencies = [ "futures", "libc", @@ -9139,7 +9139,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "ipnet", @@ -9147,7 +9147,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.5.0" +version = "2.5.1" dependencies = [ "futures-util", "if-addrs", @@ -9161,7 +9161,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9230,7 +9230,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9254,7 +9254,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9278,7 +9278,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9295,7 +9295,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9308,7 +9308,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.5.0" +version = "2.5.1" dependencies = [ "anyhow", "bincode", @@ -9326,7 +9326,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9349,7 +9349,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9420,7 +9420,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9476,7 +9476,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9545,7 +9545,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9568,7 +9568,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9591,7 +9591,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.5.0" +version = "2.5.1" dependencies = [ "bytes", "eyre", @@ -9620,7 +9620,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9631,7 +9631,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9654,7 +9654,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9665,7 +9665,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9689,7 +9689,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9698,7 +9698,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9740,7 +9740,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9794,7 +9794,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9825,7 +9825,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "arbitrary", @@ -9845,7 +9845,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9861,7 +9861,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -9941,7 +9941,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-genesis", @@ -9971,7 +9971,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-network", @@ -10029,7 +10029,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-evm", @@ -10049,7 +10049,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -10069,7 +10069,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10105,7 +10105,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10151,7 +10151,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10203,7 +10203,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-rpc-types-engine", "http", @@ -10220,7 +10220,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10250,7 +10250,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10312,7 +10312,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10346,7 +10346,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "arbitrary", @@ -10362,7 +10362,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "assert_matches", @@ -10385,7 +10385,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "clap", @@ -10403,7 +10403,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10430,7 +10430,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10446,7 +10446,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10475,7 +10475,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10504,7 +10504,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.5.0" +version = "2.5.1" dependencies = [ "crossbeam-utils", "dashmap", @@ -10524,7 +10524,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10540,7 +10540,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.5.0" +version = "2.5.1" dependencies = [ "tokio", "tokio-stream", @@ -10549,7 +10549,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.5.0" +version = "2.5.1" dependencies = [ "clap", "eyre", @@ -10568,7 +10568,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.5.0" +version = "2.5.1" dependencies = [ "base64 0.22.1", "clap", @@ -10586,7 +10586,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10637,7 +10637,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10669,7 +10669,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10701,7 +10701,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10728,7 +10728,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10756,7 +10756,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.5.0" +version = "2.5.1" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index c82dd492175..a54d01ac3fd 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.5.0" +version = "2.5.1" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index 29e1e41dae4..941f6e83ed6 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.5.0', + text: 'v2.5.1', items: [ { text: 'Releases', From 6dec1b96b625584956883c34ad0eafbe550480ac Mon Sep 17 00:00:00 2001 From: Emma Jamieson-Hoare Date: Thu, 20 Aug 2026 11:43:28 +0100 Subject: [PATCH 129/148] chore: fix formatting regression --- .../src/providers/database/provider.rs | 162 +++++++++--------- 1 file changed, 83 insertions(+), 79 deletions(-) diff --git a/crates/storage/provider/src/providers/database/provider.rs b/crates/storage/provider/src/providers/database/provider.rs index eb909288b6b..df23ce5fda3 100644 --- a/crates/storage/provider/src/providers/database/provider.rs +++ b/crates/storage/provider/src/providers/database/provider.rs @@ -314,10 +314,10 @@ impl DatabaseProvider { &'a self, mut block_number: BlockNumber, ) -> ProviderResult> { - if block_number == self.best_block_number().unwrap_or_default() - && block_number == self.last_block_number().unwrap_or_default() + if block_number == self.best_block_number().unwrap_or_default() && + block_number == self.last_block_number().unwrap_or_default() { - return Ok(Box::new(LatestStateProviderRef::new(self))); + return Ok(Box::new(LatestStateProviderRef::new(self))) } // +1 as the changeset that we want is the one that was applied after this block. @@ -540,22 +540,22 @@ impl DatabaseProvider ProviderResult { let tip = self.last_block_number()?.max(last_block); Ok(StaticFileWriteCtx { - write_senders: EitherWriterDestination::senders(self).is_static_file() - && self.prune_modes.sender_recovery.is_none_or(|m| !m.is_full()), - write_receipts: save_mode.with_state() - && EitherWriter::receipts_destination(self).is_static_file(), - write_account_changesets: save_mode.with_state() - && EitherWriterDestination::account_changesets(self).is_static_file(), - write_storage_changesets: save_mode.with_state() - && EitherWriterDestination::storage_changesets(self).is_static_file(), + write_senders: EitherWriterDestination::senders(self).is_static_file() && + self.prune_modes.sender_recovery.is_none_or(|m| !m.is_full()), + write_receipts: save_mode.with_state() && + EitherWriter::receipts_destination(self).is_static_file(), + write_account_changesets: save_mode.with_state() && + EitherWriterDestination::account_changesets(self).is_static_file(), + write_storage_changesets: save_mode.with_state() && + EitherWriterDestination::storage_changesets(self).is_static_file(), tip, receipts_prune_mode: self.prune_modes.receipts, // Receipts are prunable if no receipts exist in SF yet and within pruning distance receipts_prunable: self .static_file_provider .get_highest_static_file_tx(StaticFileSegment::Receipts) - .is_none() - && PruneMode::Distance(self.minimum_pruning_distance) + .is_none() && + PruneMode::Distance(self.minimum_pruning_distance) .should_prune(first_block, tip), }) } @@ -594,7 +594,7 @@ impl DatabaseProvider DatabaseProvider DatabaseProvider>(); + let mask = state_trie_masking_blocks + .iter() + .map(|block| block.trie_data.get().sorted.trie_updates.as_ref()) + .collect::>(); // Sparse-trie streaming emits node updates, while the serial state-root path, // including fallback, can emit a whole storage-trie wipe. `disjointed_merge_batch` // rejects wipes, so use the ordered merge. Persisting updates shadowed by the @@ -818,8 +824,8 @@ impl DatabaseProvider DatabaseProvider>, first_tx_num: TxNumber, ) -> ProviderResult { - if self.prune_modes.sender_recovery.is_none_or(|m| !m.is_full()) - && EitherWriterDestination::senders(self).is_database() + if self.prune_modes.sender_recovery.is_none_or(|m| !m.is_full()) && + EitherWriterDestination::senders(self).is_database() { let start = Instant::now(); let tx_nums_iter = std::iter::successors(Some(first_tx_num), |n| Some(n + 1)); @@ -1096,7 +1102,7 @@ where while let Some((sharded_key, list)) = item { // If the shard does not belong to the key, break. if !shard_belongs_to_key(&sharded_key) { - break; + break } // Always delete the current shard from the database first @@ -1111,18 +1117,18 @@ where // Keep it deleted (don't return anything for reinsertion) if first >= block_number { item = cursor.prev()?; - continue; + continue } // Case 2: This is a boundary shard (spans across the unwinding point) // The shard contains some blocks below and some at/above the unwinding point else if block_number <= sharded_key.as_ref().highest_block_number { // Return only the block numbers that are below the unwinding point // These will be reinserted to preserve the historical data - return Ok(list.iter().take_while(|i| *i < block_number).collect::>()); + return Ok(list.iter().take_while(|i| *i < block_number).collect::>()) } // Case 3: Entire shard is below the unwinding point // Return all block numbers for reinsertion (preserve entire shard) - return Ok(list.iter().collect::>()); + return Ok(list.iter().collect::>()) } // No shards found or all processed @@ -1201,7 +1207,7 @@ impl DatabaseProvider { let Some(block_number) = self.convert_hash_or_number(id)? else { return Ok(None) }; let earliest_available = self.static_file_provider.earliest_history_height(); if block_number < earliest_available { - return Err(ProviderError::BlockExpired { requested: block_number, earliest_available }); + return Err(ProviderError::BlockExpired { requested: block_number, earliest_available }) } let Some(header) = header_by_number(block_number)? else { return Ok(None) }; @@ -1271,7 +1277,7 @@ impl DatabaseProvider { F: FnMut(H, BodyTy, Range) -> ProviderResult, { if range.is_empty() { - return Ok(Vec::new()); + return Ok(Vec::new()) } let len = range.end().saturating_sub(*range.start()) as usize + 1; @@ -1606,8 +1612,8 @@ impl AccountExtReader for DatabaseProvider HeaderSyncGapProvider } Ordering::Less => { // There's either missing or corrupted files. - return Err(ProviderError::HeaderNotFound(next_static_file_block_num.into())); + return Err(ProviderError::HeaderNotFound(next_static_file_block_num.into())) } Ordering::Equal => {} } @@ -1940,7 +1946,7 @@ impl BlockReader for DatabaseProvid if let Some(number) = self.convert_hash_or_number(id)? { let earliest_available = self.static_file_provider.earliest_history_height(); if number < earliest_available { - return Err(ProviderError::BlockExpired { requested: number, earliest_available }); + return Err(ProviderError::BlockExpired { requested: number, earliest_available }) } let Some(header) = self.header_by_number(number)? else { return Ok(None) }; @@ -1950,7 +1956,7 @@ impl BlockReader for DatabaseProvid // If they exist but are not indexed, we don't have enough // information to return the block anyways, so we return `None`. let Some(transactions) = self.transactions_by_block(number.into())? else { - return Ok(None); + return Ok(None) }; let body = self @@ -1960,7 +1966,7 @@ impl BlockReader for DatabaseProvid .pop() .ok_or(ProviderError::InvalidStorageOutput)?; - return Ok(Some(Self::Block::new(header, body))); + return Ok(Some(Self::Block::new(header, body))) } Ok(None) @@ -2120,10 +2126,10 @@ impl TransactionsProvider for Datab &self, tx_hash: TxHash, ) -> ProviderResult> { - if let Some(transaction_id) = self.transaction_id(tx_hash)? - && let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? - && let Some(block_number) = self.block_by_transaction_id(transaction_id)? - && let Some(sealed_header) = self.sealed_header(block_number)? + if let Some(transaction_id) = self.transaction_id(tx_hash)? && + let Some(transaction) = self.transaction_by_id_unhashed(transaction_id)? && + let Some(block_number) = self.block_by_transaction_id(transaction_id)? && + let Some(sealed_header) = self.sealed_header(block_number)? { let (header, block_hash) = sealed_header.split(); if let Some(block_body) = self.block_body_indices(block_number)? { @@ -2143,7 +2149,7 @@ impl TransactionsProvider for Datab timestamp: header.timestamp(), }; - return Ok(Some((transaction, meta))); + return Ok(Some((transaction, meta))) } } @@ -2154,15 +2160,15 @@ impl TransactionsProvider for Datab &self, id: BlockHashOrNumber, ) -> ProviderResult>> { - if let Some(block_number) = self.convert_hash_or_number(id)? - && let Some(body) = self.block_body_indices(block_number)? + if let Some(block_number) = self.convert_hash_or_number(id)? && + let Some(body) = self.block_body_indices(block_number)? { let tx_range = body.tx_num_range(); return if tx_range.is_empty() { Ok(Some(Vec::new())) } else { self.transactions_by_tx_range(tx_range).map(Some) - }; + } } Ok(None) } @@ -2237,8 +2243,8 @@ impl ReceiptProvider for DatabasePr &self, block: BlockHashOrNumber, ) -> ProviderResult>> { - if let Some(number) = self.convert_hash_or_number(block)? - && let Some(body) = self.block_body_indices(number)? + if let Some(number) = self.convert_hash_or_number(block)? && + let Some(body) = self.block_body_indices(number)? { let tx_range = body.tx_num_range(); return if tx_range.is_empty() { @@ -2247,11 +2253,11 @@ impl ReceiptProvider for DatabasePr let receipts = self.receipts_by_tx_range(tx_range)?; if receipts.len() != body.tx_count as usize { - return Ok(None); + return Ok(None) } Ok(Some(receipts)) - }; + } } Ok(None) } @@ -2568,10 +2574,10 @@ impl StateWriter ) -> ProviderResult<()> { let execution_outcome = execution_outcome.into(); - if self.cached_storage_settings().use_hashed_state() - && !config.write_receipts - && !config.write_account_changesets - && !config.write_storage_changesets + if self.cached_storage_settings().use_hashed_state() && + !config.write_receipts && + !config.write_account_changesets && + !config.write_storage_changesets { // In storage v2 with all outputs directed to static files, plain state and changesets // are written elsewhere. Only bytecodes need MDBX writes, so skip the expensive @@ -2626,12 +2632,11 @@ impl StateWriter // Database, OR if receipts_in_static_files is enabled but no receipts exist in static // files yet. Once receipts exist in static files, we must continue writing to maintain // continuity and have no gaps. - let prunable_receipts = (EitherWriter::receipts_destination(self).is_database() - || self - .static_file_provider() + let prunable_receipts = (EitherWriter::receipts_destination(self).is_database() || + self.static_file_provider() .get_highest_static_file_tx(StaticFileSegment::Receipts) - .is_none()) - && PruneMode::Distance(self.minimum_pruning_distance).should_prune(first_block, tip); + .is_none()) && + PruneMode::Distance(self.minimum_pruning_distance).should_prune(first_block, tip); // Prepare set of addresses which logs should not be pruned. let mut allowed_addresses: AddressSet = AddressSet::default(); @@ -2648,13 +2653,12 @@ impl StateWriter receipts_writer.increment_block(block_number)?; // Skip writing receipts if pruning configuration requires us to. - if prunable_receipts - && self - .prune_modes + if prunable_receipts && + self.prune_modes .receipts .is_some_and(|mode| mode.should_prune(block_number, tip)) { - continue; + continue } // If there are new addresses to retain after this block number, track them @@ -2666,11 +2670,11 @@ impl StateWriter let receipt_idx = first_tx_index + idx as u64; // Skip writing receipt if log filter is active and it does not have any logs to // retain - if prunable_receipts - && has_contract_log_filter - && !receipt.logs().iter().any(|log| allowed_addresses.contains(&log.address)) + if prunable_receipts && + has_contract_log_filter && + !receipt.logs().iter().any(|log| allowed_addresses.contains(&log.address)) { - continue; + continue } receipts_writer.append_receipt(receipt_idx, receipt)?; @@ -2801,8 +2805,8 @@ impl StateWriter for entry in storage { tracing::trace!(?address, ?entry.key, "Updating plain state storage"); if let Some(db_entry) = - storages_cursor.seek_by_key_subkey(address, entry.key)? - && db_entry.key == entry.key + storages_cursor.seek_by_key_subkey(address, entry.key)? && + db_entry.key == entry.key { storages_cursor.delete_current()?; } @@ -2848,8 +2852,8 @@ impl StateWriter let entry = StorageEntry { key: *hashed_slot, value: *value }; if let Some(db_entry) = - hashed_storage_cursor.seek_by_key_subkey(*hashed_address, entry.key)? - && db_entry.key == entry.key + hashed_storage_cursor.seek_by_key_subkey(*hashed_address, entry.key)? && + db_entry.key == entry.key { hashed_storage_cursor.delete_current()?; } @@ -3032,7 +3036,7 @@ impl StateWriter let range = block + 1..=self.last_block_number()?; if range.is_empty() { - return Ok(ExecutionOutcome::default()); + return Ok(ExecutionOutcome::default()) } let start_block_number = *range.start(); @@ -3048,8 +3052,8 @@ impl StateWriter let storage_range = BlockNumberAddress::range(range.clone()); let storage_changeset = if let Some(highest_block) = self .static_file_provider - .get_highest_static_file_block(StaticFileSegment::StorageChangeSets) - && self.cached_storage_settings().storage_v2 + .get_highest_static_file_block(StaticFileSegment::StorageChangeSets) && + self.cached_storage_settings().storage_v2 { let changesets = self.storage_changesets_range(block + 1..=highest_block)?; let mut changeset_writer = @@ -3064,8 +3068,8 @@ impl StateWriter let highest_changeset_block = self .static_file_provider .get_highest_static_file_block(StaticFileSegment::AccountChangeSets); - let account_changeset = if let Some(highest_block) = highest_changeset_block - && self.cached_storage_settings().storage_v2 + let account_changeset = if let Some(highest_block) = highest_changeset_block && + self.cached_storage_settings().storage_v2 { // TODO: add a `take` method that removes and returns the items instead of doing this let changesets = self.account_changesets_range(block + 1..highest_block + 1)?; @@ -3272,7 +3276,7 @@ impl TrieWriter for DatabaseProvider #[instrument(level = "debug", target = "providers::db", skip_all)] fn write_trie_updates_sorted(&self, trie_updates: &TrieUpdatesSorted) -> ProviderResult { if trie_updates.is_empty() { - return Ok(0); + return Ok(0) } // Track the number of inserted entries. @@ -3530,8 +3534,8 @@ impl HistoryWriter for DatabaseProvi StorageShardedKey::last(address, storage_key), rem_index, |storage_sharded_key| { - storage_sharded_key.address == address - && storage_sharded_key.sharded_key.key == storage_key + storage_sharded_key.address == address && + storage_sharded_key.sharded_key.key == storage_key }, )?; @@ -3831,7 +3835,7 @@ impl BlockWriter ) -> ProviderResult<()> { if blocks.is_empty() { debug!(target: "providers::db", "Attempted to append empty block range"); - return Ok(()); + return Ok(()) } // Blocks are not empty, so no need to handle the case of `blocks.first()` being @@ -5671,8 +5675,8 @@ mod tests { ..Default::default() }; - let storage: HashMap> = (1 - ..=slots_per_account as u64) + let storage: HashMap> = (1..= + slots_per_account as u64) .map(|s| { ( U256::from(s + acct_idx as u64 * 100), From 29aa4aeb76bc273ba3e75527f329113bcc90f284 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Tue, 1 Sep 2026 18:18:14 +0000 Subject: [PATCH 130/148] fix(stages): log bad block hash (#26909) Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com> --- crates/engine/tree/src/tree/precompile_cache.rs | 9 +++++++-- crates/stages/api/src/pipeline/mod.rs | 2 ++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/crates/engine/tree/src/tree/precompile_cache.rs b/crates/engine/tree/src/tree/precompile_cache.rs index e868cca8485..23146ca3914 100644 --- a/crates/engine/tree/src/tree/precompile_cache.rs +++ b/crates/engine/tree/src/tree/precompile_cache.rs @@ -14,6 +14,9 @@ use tracing::error; /// Default max cache size for [`PrecompileCache`] const MAX_CACHE_SIZE: u32 = 1024 * 1024; +/// Maximum calldata size to cache for a precompile. +const MAX_PRECOMPILE_CACHE_INPUT_SIZE: usize = 2 * 1024; + /// Stores caches for each precompile. #[derive(Debug, Clone, Default)] pub struct PrecompileCacheMap(Arc, FbBuildHasher<20>>>) @@ -179,7 +182,9 @@ where } fn call(&self, input: PrecompileInput<'_>) -> PrecompileResult { - if let Some(entry) = &self.cache.get(input.data, self.spec_id.clone()) && + let cacheable_input = input.data.len() <= MAX_PRECOMPILE_CACHE_INPUT_SIZE; + if cacheable_input && + let Some(entry) = &self.cache.get(input.data, self.spec_id.clone()) && input.gas >= entry.gas_used() { self.increment_by_one_precompile_cache_hits(); @@ -193,7 +198,7 @@ where match &result { // Only successful outputs are cacheable. Non-success statuses and errors must execute // again instead of poisoning the cache for subsequent calls. - Ok(output) if output.is_success() => { + Ok(output) if cacheable_input && output.is_success() => { // Sanity-check precompile output to ensure that it does not affect state gas in any // way. // diff --git a/crates/stages/api/src/pipeline/mod.rs b/crates/stages/api/src/pipeline/mod.rs index 0c2f3b668d6..36a8a6f3c3d 100644 --- a/crates/stages/api/src/pipeline/mod.rs +++ b/crates/stages/api/src/pipeline/mod.rs @@ -585,6 +585,7 @@ impl Pipeline { target: "sync::pipeline", stage = %stage_id, bad_block = %block.block.number, + bad_block_hash = %block.block.hash, "Stage encountered a validation error: {validation_error}" ); @@ -609,6 +610,7 @@ impl Pipeline { target: "sync::pipeline", stage = %stage_id, bad_block = %block.block.number, + bad_block_hash = %block.block.hash, "Stage encountered an execution error: {execution_error}" ); } From 60c1eb75d809fe26719fc52ad7e6166f9e5efc68 Mon Sep 17 00:00:00 2001 From: Sergei Shulepov Date: Tue, 25 Aug 2026 22:19:09 +0000 Subject: [PATCH 131/148] fix(rocksdb): better max_open_files config (#26812) --- Cargo.lock | 1 + crates/storage/provider/Cargo.toml | 3 + .../src/providers/rocksdb/provider.rs | 84 +++++++++++++++++-- 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 04c928a3f41..00bed560e7d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9751,6 +9751,7 @@ dependencies = [ "assert_matches", "eyre", "itertools 0.14.0", + "libc", "metrics", "notify", "parking_lot", diff --git a/crates/storage/provider/Cargo.toml b/crates/storage/provider/Cargo.toml index 1ff298889c1..17db3b470e5 100644 --- a/crates/storage/provider/Cargo.toml +++ b/crates/storage/provider/Cargo.toml @@ -69,6 +69,9 @@ rayon.workspace = true rocksdb.workspace = true +[target.'cfg(unix)'.dependencies] +libc.workspace = true + [dev-dependencies] reth-db = { workspace = true, features = ["test-utils"] } reth-primitives-traits = { workspace = true, features = ["arbitrary", "test-utils"] } diff --git a/crates/storage/provider/src/providers/rocksdb/provider.rs b/crates/storage/provider/src/providers/rocksdb/provider.rs index d7198e7c74a..9fae9514ebd 100644 --- a/crates/storage/provider/src/providers/rocksdb/provider.rs +++ b/crates/storage/provider/src/providers/rocksdb/provider.rs @@ -115,14 +115,22 @@ const DEFAULT_BLOCK_SIZE: usize = 16 * 1024; /// Default max background jobs for `RocksDB` compaction and flushing. const DEFAULT_MAX_BACKGROUND_JOBS: i32 = 6; -/// Default max open file descriptors for `RocksDB`. +/// Max open file descriptors for `RocksDB` when the process limit is low. /// /// Caps the number of SST file handles `RocksDB` keeps open simultaneously. /// Set to 512 to stay within the common default OS `ulimit -n` of 1024, /// leaving headroom for MDBX, static files, and other I/O. -/// `RocksDB` uses an internal table cache and re-opens files on demand, -/// so this has negligible performance impact on read-heavy workloads. -const DEFAULT_MAX_OPEN_FILES: i32 = 512; +const LIMITED_MAX_OPEN_FILES: i32 = 512; + +/// Keeps all `RocksDB` files open and avoids table cache lookups. +const KEEP_ALL_FILES_OPEN: i32 = -1; + +/// Minimum process file descriptor limit for keeping all `RocksDB` files open. +/// +/// Mature archive databases can use tens of thousands of file descriptors. This threshold keeps +/// unlimited mode for hosts configured for that workload while retaining a bounded table cache on +/// systems with low limits. +const HIGH_FILE_DESCRIPTOR_LIMIT: u64 = 128 * 1024; /// Default bytes per sync for `RocksDB` WAL writes (1 MB). const DEFAULT_BYTES_PER_SYNC: u64 = 1_048_576; @@ -234,7 +242,7 @@ impl RocksDBBuilder { options.set_log_level(log_level); - options.set_max_open_files(DEFAULT_MAX_OPEN_FILES); + options.set_max_open_files(select_max_open_files()); // Delete obsolete WAL files immediately after all column families have flushed. // Both set to 0 means "delete ASAP, no archival". @@ -415,7 +423,7 @@ impl RocksDBBuilder { // Open as secondary instance for catch-up capability. // Secondary needs max_open_files = -1 to keep all FDs open. let mut options = options; - options.set_max_open_files(-1); + options.set_max_open_files(KEEP_ALL_FILES_OPEN); let secondary_path = self .path @@ -2879,6 +2887,60 @@ const fn convert_log_level(level: LogLevel) -> rocksdb::LogLevel { } } +/// Selects the `RocksDB` `max_open_files` setting from the current file descriptor limit +/// balancing performance vs. compatibility. +/// +/// A mature database can use tens of thousands of file descriptors. With a +/// finite `max_open_files` value, `RocksDB` performs additional table-cache checks +/// even when enough capacity is available. The value `-1` keeps all table files +/// open and avoids this overhead. +/// +/// During normal startup, Reth raises the soft file descriptor limit to the +/// system hard limit before it opens `RocksDB`. Therefore, we expect the current +/// limit to be the hard limit. A survey of modern Linux distributions shows +/// common default limits of `1024:524288`. +/// +/// We use `-1` only when the current limit is above a conservative threshold. +/// This keeps enough file descriptors for other parts of the process. When the +/// limit is lower or cannot be read, we use a stricter fixed limit. +fn select_max_open_files() -> i32 { + let file_descriptor_limit = current_file_descriptor_limit(); + let max_open_files = max_open_files_for_limit(file_descriptor_limit); + + if max_open_files == LIMITED_MAX_OPEN_FILES { + tracing::warn!( + target: "providers::rocksdb", + ?file_descriptor_limit, + threshold = HIGH_FILE_DESCRIPTOR_LIMIT, + max_open_files, + "RocksDB will not keep all files open; performance may be reduced" + ); + } + + max_open_files +} + +const fn max_open_files_for_limit(file_descriptor_limit: Option) -> i32 { + match file_descriptor_limit { + Some(limit) if limit >= HIGH_FILE_DESCRIPTOR_LIMIT => KEEP_ALL_FILES_OPEN, + _ => LIMITED_MAX_OPEN_FILES, + } +} + +#[cfg(unix)] +#[allow(clippy::useless_conversion)] +fn current_file_descriptor_limit() -> Option { + let mut limit = libc::rlimit { rlim_cur: 0, rlim_max: 0 }; + // SAFETY: `limit` points to initialized writable memory for the kernel to populate. + let result = unsafe { libc::getrlimit(libc::RLIMIT_NOFILE, &raw mut limit) }; + (result == 0).then(|| limit.rlim_cur.into()) +} + +#[cfg(not(unix))] +const fn current_file_descriptor_limit() -> Option { + None +} + #[cfg(test)] mod tests { use super::*; @@ -2895,6 +2957,16 @@ mod tests { }; use tempfile::TempDir; + #[test] + fn max_open_files_adapts_to_file_descriptor_limit() { + assert_eq!(max_open_files_for_limit(Some(HIGH_FILE_DESCRIPTOR_LIMIT)), KEEP_ALL_FILES_OPEN); + assert_eq!( + max_open_files_for_limit(Some(HIGH_FILE_DESCRIPTOR_LIMIT - 1)), + LIMITED_MAX_OPEN_FILES + ); + assert_eq!(max_open_files_for_limit(None), LIMITED_MAX_OPEN_FILES); + } + #[test] fn test_with_default_tables_registers_required_column_families() { let temp_dir = TempDir::new().unwrap(); From 8eacb192fe01d4834e52e9d3003e625690eb4945 Mon Sep 17 00:00:00 2001 From: refcell Date: Tue, 25 Aug 2026 15:25:25 +0000 Subject: [PATCH 132/148] feat(net): allow updating the fork filter at runtime (#26794) Co-authored-by: Claude Co-authored-by: Matthias Seitz --- crates/net/network/src/manager.rs | 4 ++++ crates/net/network/src/network.rs | 24 +++++++++++++++++++++++- crates/net/network/src/session/mod.rs | 17 +++++++++++++++++ crates/net/p2p/src/sync.rs | 13 ++++++++++++- 4 files changed, 56 insertions(+), 2 deletions(-) diff --git a/crates/net/network/src/manager.rs b/crates/net/network/src/manager.rs index 515b27d690b..63255c41daa 100644 --- a/crates/net/network/src/manager.rs +++ b/crates/net/network/src/manager.rs @@ -776,6 +776,10 @@ impl NetworkManager { self.swarm.state_mut().update_fork_id(transition.current); } } + NetworkHandleMessage::SetForkFilter { fork_filter } => { + let fork_id = self.swarm.sessions_mut().set_fork_filter(fork_filter); + self.swarm.state_mut().update_fork_id(fork_id); + } NetworkHandleMessage::GetPeerInfos(tx) => { let _ = tx.send(self.get_peer_infos()); } diff --git a/crates/net/network/src/network.rs b/crates/net/network/src/network.rs index 18f381b3cbf..0ad5356bfee 100644 --- a/crates/net/network/src/network.rs +++ b/crates/net/network/src/network.rs @@ -12,7 +12,7 @@ use reth_eth_wire::{ BlockRangeUpdate, BroadcastPoolTransactions, DisconnectReason, EthNetworkPrimitives, NetworkPrimitives, NewPooledTransactionHashes, SharedTransactions, }; -use reth_ethereum_forks::Head; +use reth_ethereum_forks::{ForkFilter, Head}; use reth_network_api::{ events::{NetworkPeersEvents, PeerEvent, PeerEventStream}, test_utils::{PeersHandle, PeersHandleProvider}, @@ -112,6 +112,18 @@ impl NetworkHandle { self.send_message(NetworkHandleMessage::StatusUpdate { head }); } + /// Replaces the network's active [`ForkFilter`] with `fork_filter`, re-deriving the advertised + /// [`ForkId`](reth_ethereum_forks::ForkId) for future handshakes and updating the discovery + /// ENR entry. + /// + /// This lets a running node adopt a fork schedule that changed at runtime (e.g. an + /// L1-signalled upgrade) without a restart. The caller must build `fork_filter` from the + /// updated chain spec advanced to the node's current head, and should do so before the fork's + /// activation timestamp so the node announces the upcoming fork ahead of time. + pub fn set_fork_filter(&self, fork_filter: ForkFilter) { + self.send_message(NetworkHandleMessage::SetForkFilter { fork_filter }); + } + /// Announce a block over devp2p /// /// Caution: in `PoS` this is a noop because new blocks are no longer announced over devp2p. @@ -497,6 +509,11 @@ impl NetworkSyncUpdater for NetworkHandle { fn update_block_range(&self, update: reth_eth_wire::BlockRangeUpdate) { self.send_message(NetworkHandleMessage::InternalBlockRangeUpdate(update)); } + + /// Replaces the active fork filter to adopt a runtime fork-schedule change. + fn set_fork_filter(&self, fork_filter: ForkFilter) { + self.send_message(NetworkHandleMessage::SetForkFilter { fork_filter }); + } } impl BlockDownloaderProvider for NetworkHandle { @@ -614,6 +631,11 @@ pub(crate) enum NetworkHandleMessage), /// Gets `PeerInfo` for the specified peer IDs. diff --git a/crates/net/network/src/session/mod.rs b/crates/net/network/src/session/mod.rs index db95a71d984..a70a72742f4 100644 --- a/crates/net/network/src/session/mod.rs +++ b/crates/net/network/src/session/mod.rs @@ -263,6 +263,23 @@ impl SessionManager { transition } + /// Replaces the active [`ForkFilter`] with a newly derived one and returns the resulting + /// [`ForkId`]. + /// + /// This is used to adopt a fork schedule that changed at runtime (e.g. an L1-signalled + /// upgrade) without restarting the node. The caller is expected to build `fork_filter` from + /// the updated chain spec already advanced to the node's current head, so that + /// [`ForkFilter::current`] reflects the correct [`ForkId`] immediately. + /// + /// Every subsequent handshake clones this filter and advertises the returned [`ForkId`], so + /// the node's advertised fork identity stays aligned with the rules it now enforces. Existing + /// active sessions are not revalidated here; see [`SessionManager::active_sessions`]. + pub(crate) fn set_fork_filter(&mut self, fork_filter: ForkFilter) -> ForkId { + self.fork_filter = fork_filter; + self.status.forkid = self.fork_filter.current(); + self.status.forkid + } + /// An incoming TCP connection was received. This starts the authentication process to turn this /// stream into an active peer session. /// diff --git a/crates/net/p2p/src/sync.rs b/crates/net/p2p/src/sync.rs index 77b97116d19..6770c5d12a2 100644 --- a/crates/net/p2p/src/sync.rs +++ b/crates/net/p2p/src/sync.rs @@ -1,6 +1,6 @@ //! Traits used when interacting with the sync status of the network. -use alloy_eips::eip2124::Head; +use alloy_eips::eip2124::{ForkFilter, Head}; use reth_eth_wire_types::BlockRangeUpdate; /// A type that provides information about whether the node is currently syncing and the network is @@ -31,6 +31,17 @@ pub trait NetworkSyncUpdater: std::fmt::Debug + Send + Sync + 'static { /// Updates the advertised block range. fn update_block_range(&self, update: BlockRangeUpdate); + + /// Replaces the node's active [`ForkFilter`] to adopt a fork schedule that changed at runtime + /// (e.g. an L1-signalled upgrade) without a restart. + /// + /// The caller must build `fork_filter` from the updated chain spec advanced to the node's + /// current head, and should do so before the fork's activation timestamp so the node + /// announces the upcoming fork ahead of time. Defaults to a no-op for updaters that do not + /// track a fork filter. + fn set_fork_filter(&self, fork_filter: ForkFilter) { + let _ = fork_filter; + } } /// The state the network is currently in when it comes to synchronization. From b5f0c4f2d0eb981afa2c063acbdd605cb419228b Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Wed, 26 Aug 2026 11:50:31 +0000 Subject: [PATCH 133/148] fix(exex): support sparse WAL file IDs (#26819) --- crates/exex/exex/src/wal/mod.rs | 68 ++++++++++++++++++++++------- crates/exex/exex/src/wal/storage.rs | 34 ++++++--------- 2 files changed, 67 insertions(+), 35 deletions(-) diff --git a/crates/exex/exex/src/wal/mod.rs b/crates/exex/exex/src/wal/mod.rs index 0836e15b55a..e96d8b6cd3e 100644 --- a/crates/exex/exex/src/wal/mod.rs +++ b/crates/exex/exex/src/wal/mod.rs @@ -113,13 +113,14 @@ where /// Fills the block cache with the notifications from the storage. #[instrument(skip(self))] fn fill_block_cache(&self) -> WalResult<()> { - let Some(files_range) = self.storage.files_range()? else { return Ok(()) }; - self.next_file_id.store(files_range.end() + 1, Ordering::Relaxed); + let file_ids = self.storage.file_ids()?; + let Some(last_file_id) = file_ids.last() else { return Ok(()) }; + self.next_file_id.store(last_file_id + 1, Ordering::Relaxed); let mut block_cache = self.block_cache.write(); let mut notifications_size = 0; - for entry in self.storage.iter_notifications(files_range) { + for entry in self.storage.iter_notifications(file_ids) { let (file_id, size, notification) = entry?; notifications_size += size; @@ -198,11 +199,9 @@ where fn iter_notifications( &self, ) -> WalResult>> + '_>> { - let Some(range) = self.storage.files_range()? else { - return Ok(Box::new(std::iter::empty())) - }; - - Ok(Box::new(self.storage.iter_notifications(range).map(|entry| Ok(entry?.2)))) + Ok(Box::new( + self.storage.iter_notifications(self.storage.file_ids()?).map(|entry| Ok(entry?.2)), + )) } } @@ -238,6 +237,7 @@ mod tests { use crate::wal::{cache::CachedBlock, error::WalResult, Wal}; use alloy_primitives::B256; use itertools::Itertools; + use reth_ethereum_primitives::EthPrimitives; use reth_exex_types::ExExNotification; use reth_provider::Chain; use reth_testing_utils::generators::{ @@ -246,13 +246,11 @@ mod tests { use std::{collections::BTreeMap, sync::Arc}; fn read_notifications(wal: &Wal) -> WalResult> { - wal.inner.storage.files_range()?.map_or(Ok(Vec::new()), |range| { - wal.inner - .storage - .iter_notifications(range) - .map(|entry| entry.map(|(_, _, n)| n)) - .collect() - }) + wal.inner + .storage + .iter_notifications(wal.inner.storage.file_ids()?) + .map(|entry| entry.map(|(_, _, n)| n)) + .collect() } fn sort_committed_blocks( @@ -522,4 +520,44 @@ mod tests { Ok(()) } + + #[test] + fn test_finalize_with_sparse_file_ids() -> eyre::Result<()> { + let mut rng = generators::rng(); + + let temp_dir = tempfile::tempdir()?; + let wal = Wal::::new(&temp_dir)?; + + let blocks = random_block_range(&mut rng, 0..=5, BlockRangeParams::default()) + .into_iter() + .map(|block| block.try_recover()) + .collect::, _>>()?; + let chain = |range: std::ops::RangeInclusive| { + Arc::new(Chain::new(blocks[range].to_vec(), Default::default(), BTreeMap::new())) + }; + + let commit_to_four = ExExNotification::ChainCommitted { new: chain(1..=4) }; + let revert_to_two = ExExNotification::ChainReverted { old: chain(2..=4) }; + let commit_two = ExExNotification::ChainCommitted { new: chain(2..=2) }; + let commit_to_five = ExExNotification::ChainCommitted { new: chain(3..=5) }; + + wal.commit(&commit_to_four)?; + wal.commit(&revert_to_two)?; + wal.commit(&commit_two)?; + wal.commit(&commit_to_five)?; + assert_eq!(wal.inner.storage.file_ids()?, vec![0, 1, 2, 3]); + + // File 2 has a lower maximum block than the surrounding notifications, so finalizing it + // leaves a gap in the file ID sequence. + wal.finalize((blocks[2].number, blocks[2].hash()).into())?; + assert_eq!(wal.inner.storage.file_ids()?, vec![0, 1, 3]); + + let wal = Wal::::new(&temp_dir)?; + assert_eq!(read_notifications(&wal)?, vec![commit_to_four, revert_to_two, commit_to_five]); + + wal.commit(&commit_two)?; + assert_eq!(wal.inner.storage.file_ids()?, vec![0, 1, 3, 4]); + + Ok(()) + } } diff --git a/crates/exex/exex/src/wal/storage.rs b/crates/exex/exex/src/wal/storage.rs index 8c36e3f624d..45363130c23 100644 --- a/crates/exex/exex/src/wal/storage.rs +++ b/crates/exex/exex/src/wal/storage.rs @@ -1,6 +1,5 @@ use std::{ fs::File, - ops::RangeInclusive, path::{Path, PathBuf}, }; @@ -69,12 +68,9 @@ where } } - /// Returns the range of file IDs in the storage. - /// - /// If there are no files in the storage, returns `None`. - pub(super) fn files_range(&self) -> WalResult>> { - let mut min_id = None; - let mut max_id = None; + /// Returns the file IDs in the storage in ascending order. + pub(super) fn file_ids(&self) -> WalResult> { + let mut file_ids = Vec::new(); for entry in reth_fs_util::read_dir(&self.path)? { let entry = entry.map_err(|err| WalError::DirEntry(self.path.clone(), err))?; @@ -82,13 +78,12 @@ where if entry.path().extension() == Some(FILE_EXTENSION.as_ref()) { let file_name = entry.file_name(); let file_id = Self::parse_filename(&file_name.to_string_lossy())?; - - min_id = min_id.map_or(Some(file_id), |min_id: u32| Some(min_id.min(file_id))); - max_id = max_id.map_or(Some(file_id), |max_id: u32| Some(max_id.max(file_id))); + file_ids.push(file_id); } } - Ok(min_id.zip(max_id).map(|(min_id, max_id)| min_id..=max_id)) + file_ids.sort_unstable(); + Ok(file_ids) } /// Removes notifications from the storage according to the given list of file IDs. @@ -113,11 +108,11 @@ where Ok((deleted_total, deleted_size)) } - pub(super) fn iter_notifications( - &self, - range: RangeInclusive, - ) -> impl Iterator)>> + '_ { - range.map(move |id| { + pub(super) fn iter_notifications<'a>( + &'a self, + file_ids: impl IntoIterator + 'a, + ) -> impl Iterator)>> + 'a { + file_ids.into_iter().map(move |id| { let (notification, size) = self.read_notification(id)?.ok_or(WalError::FileNotFound(id))?; @@ -316,21 +311,20 @@ mod tests { } #[test] - fn test_files_range() -> eyre::Result<()> { + fn test_file_ids() -> eyre::Result<()> { let temp_dir = tempfile::tempdir()?; let storage: Storage = Storage::new(&temp_dir)?; // Create WAL files File::create(storage.file_path(1))?; - File::create(storage.file_path(2))?; File::create(storage.file_path(3))?; // Create non-WAL files that should be ignored File::create(temp_dir.path().join("0.tmp"))?; File::create(temp_dir.path().join("4.tmp"))?; - // Check files range - assert_eq!(storage.files_range()?, Some(1..=3)); + // Check existing file IDs are returned in order without filling the gap + assert_eq!(storage.file_ids()?, vec![1, 3]); Ok(()) } From f4b39f98108aa684c079cf073240bbe2798d2618 Mon Sep 17 00:00:00 2001 From: Emma Jamieson-Hoare Date: Wed, 2 Sep 2026 11:34:11 +0100 Subject: [PATCH 134/148] chore: bump version --- Cargo.lock | 228 +++++++++++++++++++-------------------- Cargo.toml | 62 ++++++++--- docs/vocs/vocs.config.ts | 2 +- 3 files changed, 162 insertions(+), 130 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 00bed560e7d..016f891a405 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3269,7 +3269,7 @@ dependencies = [ [[package]] name = "ef-test-runner" -version = "2.5.1" +version = "2.5.2" dependencies = [ "clap", "ef-tests", @@ -3277,7 +3277,7 @@ dependencies = [ [[package]] name = "ef-tests" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -3730,7 +3730,7 @@ dependencies = [ [[package]] name = "example-full-contract-state" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "eyre", @@ -3848,7 +3848,7 @@ dependencies = [ [[package]] name = "exex-subscription" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "clap", @@ -7661,7 +7661,7 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "reth" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-node-bindings", "alloy-primitives", @@ -7711,7 +7711,7 @@ dependencies = [ [[package]] name = "reth-basic-payload-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7737,7 +7737,7 @@ dependencies = [ [[package]] name = "reth-bb" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7784,7 +7784,7 @@ dependencies = [ [[package]] name = "reth-chain-state" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -7815,7 +7815,7 @@ dependencies = [ [[package]] name = "reth-chainspec" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7835,7 +7835,7 @@ dependencies = [ [[package]] name = "reth-cli" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-genesis", "clap", @@ -7847,7 +7847,7 @@ dependencies = [ [[package]] name = "reth-cli-commands" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -7937,7 +7937,7 @@ dependencies = [ [[package]] name = "reth-cli-runner" -version = "2.5.1" +version = "2.5.2" dependencies = [ "reth-tasks", "tokio", @@ -7946,7 +7946,7 @@ dependencies = [ [[package]] name = "reth-cli-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8000,7 +8000,7 @@ dependencies = [ [[package]] name = "reth-config" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "eyre", @@ -8018,7 +8018,7 @@ dependencies = [ [[package]] name = "reth-consensus" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8031,7 +8031,7 @@ dependencies = [ [[package]] name = "reth-consensus-common" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8045,7 +8045,7 @@ dependencies = [ [[package]] name = "reth-consensus-debug-client" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8069,7 +8069,7 @@ dependencies = [ [[package]] name = "reth-db" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8105,7 +8105,7 @@ dependencies = [ [[package]] name = "reth-db-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8133,7 +8133,7 @@ dependencies = [ [[package]] name = "reth-db-common" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-genesis", @@ -8164,7 +8164,7 @@ dependencies = [ [[package]] name = "reth-db-models" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8180,7 +8180,7 @@ dependencies = [ [[package]] name = "reth-discv4" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8206,7 +8206,7 @@ dependencies = [ [[package]] name = "reth-discv5" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -8231,7 +8231,7 @@ dependencies = [ [[package]] name = "reth-dns-discovery" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-primitives", @@ -8259,7 +8259,7 @@ dependencies = [ [[package]] name = "reth-downloaders" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8297,7 +8297,7 @@ dependencies = [ [[package]] name = "reth-e2e-test-utils" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8353,7 +8353,7 @@ dependencies = [ [[package]] name = "reth-ecies" -version = "2.5.1" +version = "2.5.2" dependencies = [ "aes", "alloy-primitives", @@ -8379,7 +8379,7 @@ dependencies = [ [[package]] name = "reth-engine-local" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8401,7 +8401,7 @@ dependencies = [ [[package]] name = "reth-engine-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8427,7 +8427,7 @@ dependencies = [ [[package]] name = "reth-engine-tree" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8497,7 +8497,7 @@ dependencies = [ [[package]] name = "reth-engine-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8526,7 +8526,7 @@ dependencies = [ [[package]] name = "reth-era" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8551,7 +8551,7 @@ dependencies = [ [[package]] name = "reth-era-downloader" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "bytes", @@ -8569,7 +8569,7 @@ dependencies = [ [[package]] name = "reth-era-utils" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -8600,7 +8600,7 @@ dependencies = [ [[package]] name = "reth-errors" -version = "2.5.1" +version = "2.5.2" dependencies = [ "reth-consensus", "reth-execution-errors", @@ -8610,7 +8610,7 @@ dependencies = [ [[package]] name = "reth-eth-wire" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8648,7 +8648,7 @@ dependencies = [ [[package]] name = "reth-eth-wire-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -8676,7 +8676,7 @@ dependencies = [ [[package]] name = "reth-ethereum" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-rpc-types-engine", "alloy-rpc-types-eth", @@ -8716,7 +8716,7 @@ dependencies = [ [[package]] name = "reth-ethereum-cli" -version = "2.5.1" +version = "2.5.2" dependencies = [ "clap", "eyre", @@ -8739,7 +8739,7 @@ dependencies = [ [[package]] name = "reth-ethereum-consensus" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8755,7 +8755,7 @@ dependencies = [ [[package]] name = "reth-ethereum-engine-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -8771,7 +8771,7 @@ dependencies = [ [[package]] name = "reth-ethereum-forks" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eip2124", "alloy-hardforks 0.4.7", @@ -8784,7 +8784,7 @@ dependencies = [ [[package]] name = "reth-ethereum-payload-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8813,7 +8813,7 @@ dependencies = [ [[package]] name = "reth-ethereum-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8834,7 +8834,7 @@ dependencies = [ [[package]] name = "reth-etl" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "rayon", @@ -8844,7 +8844,7 @@ dependencies = [ [[package]] name = "reth-evm" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -8870,7 +8870,7 @@ dependencies = [ [[package]] name = "reth-evm-ethereum" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8895,7 +8895,7 @@ dependencies = [ [[package]] name = "reth-execution-cache" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "fixed-cache", @@ -8913,7 +8913,7 @@ dependencies = [ [[package]] name = "reth-execution-errors" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-evm", "alloy-primitives", @@ -8925,7 +8925,7 @@ dependencies = [ [[package]] name = "reth-execution-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8946,7 +8946,7 @@ dependencies = [ [[package]] name = "reth-exex" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -8991,7 +8991,7 @@ dependencies = [ [[package]] name = "reth-exex-test-utils" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "eyre", @@ -9022,7 +9022,7 @@ dependencies = [ [[package]] name = "reth-exex-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -9039,7 +9039,7 @@ dependencies = [ [[package]] name = "reth-fs-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "serde", "serde_json", @@ -9048,7 +9048,7 @@ dependencies = [ [[package]] name = "reth-invalid-block-hooks" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9079,7 +9079,7 @@ dependencies = [ [[package]] name = "reth-ipc" -version = "2.5.1" +version = "2.5.2" dependencies = [ "bytes", "futures", @@ -9101,7 +9101,7 @@ dependencies = [ [[package]] name = "reth-libmdbx" -version = "2.5.1" +version = "2.5.2" dependencies = [ "bitflags 2.13.0", "byteorder", @@ -9118,7 +9118,7 @@ dependencies = [ [[package]] name = "reth-mdbx-sys" -version = "2.5.1" +version = "2.5.2" dependencies = [ "bindgen", "cc", @@ -9126,7 +9126,7 @@ dependencies = [ [[package]] name = "reth-metrics" -version = "2.5.1" +version = "2.5.2" dependencies = [ "futures", "libc", @@ -9139,7 +9139,7 @@ dependencies = [ [[package]] name = "reth-net-banlist" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "ipnet", @@ -9147,7 +9147,7 @@ dependencies = [ [[package]] name = "reth-net-nat" -version = "2.5.1" +version = "2.5.2" dependencies = [ "futures-util", "if-addrs", @@ -9161,7 +9161,7 @@ dependencies = [ [[package]] name = "reth-network" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9230,7 +9230,7 @@ dependencies = [ [[package]] name = "reth-network-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9254,7 +9254,7 @@ dependencies = [ [[package]] name = "reth-network-p2p" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9278,7 +9278,7 @@ dependencies = [ [[package]] name = "reth-network-peers" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "alloy-rlp", @@ -9295,7 +9295,7 @@ dependencies = [ [[package]] name = "reth-network-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eip2124", "humantime-serde", @@ -9308,7 +9308,7 @@ dependencies = [ [[package]] name = "reth-nippy-jar" -version = "2.5.1" +version = "2.5.2" dependencies = [ "anyhow", "bincode", @@ -9326,7 +9326,7 @@ dependencies = [ [[package]] name = "reth-node-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-rpc-types-engine", "eyre", @@ -9349,7 +9349,7 @@ dependencies = [ [[package]] name = "reth-node-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9420,7 +9420,7 @@ dependencies = [ [[package]] name = "reth-node-core" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9476,7 +9476,7 @@ dependencies = [ [[package]] name = "reth-node-ethereum" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-contract", @@ -9545,7 +9545,7 @@ dependencies = [ [[package]] name = "reth-node-ethstats" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9568,7 +9568,7 @@ dependencies = [ [[package]] name = "reth-node-events" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9591,7 +9591,7 @@ dependencies = [ [[package]] name = "reth-node-metrics" -version = "2.5.1" +version = "2.5.2" dependencies = [ "bytes", "eyre", @@ -9620,7 +9620,7 @@ dependencies = [ [[package]] name = "reth-node-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "reth-chainspec", "reth-db-api", @@ -9631,7 +9631,7 @@ dependencies = [ [[package]] name = "reth-payload-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9654,7 +9654,7 @@ dependencies = [ [[package]] name = "reth-payload-builder-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "pin-project", "reth-payload-primitives", @@ -9665,7 +9665,7 @@ dependencies = [ [[package]] name = "reth-payload-primitives" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -9689,7 +9689,7 @@ dependencies = [ [[package]] name = "reth-payload-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9698,7 +9698,7 @@ dependencies = [ [[package]] name = "reth-payload-validator" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-rpc-types-engine", @@ -9740,7 +9740,7 @@ dependencies = [ [[package]] name = "reth-provider" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -9795,7 +9795,7 @@ dependencies = [ [[package]] name = "reth-prune" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9826,7 +9826,7 @@ dependencies = [ [[package]] name = "reth-prune-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "arbitrary", @@ -9846,7 +9846,7 @@ dependencies = [ [[package]] name = "reth-revm" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -9862,7 +9862,7 @@ dependencies = [ [[package]] name = "reth-rpc" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -9942,7 +9942,7 @@ dependencies = [ [[package]] name = "reth-rpc-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-genesis", @@ -9972,7 +9972,7 @@ dependencies = [ [[package]] name = "reth-rpc-builder" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-network", @@ -10030,7 +10030,7 @@ dependencies = [ [[package]] name = "reth-rpc-convert" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-evm", @@ -10050,7 +10050,7 @@ dependencies = [ [[package]] name = "reth-rpc-e2e-tests" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-genesis", "alloy-rpc-types-engine", @@ -10070,7 +10070,7 @@ dependencies = [ [[package]] name = "reth-rpc-engine-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10106,7 +10106,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-dyn-abi", @@ -10152,7 +10152,7 @@ dependencies = [ [[package]] name = "reth-rpc-eth-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-chains", "alloy-consensus", @@ -10204,7 +10204,7 @@ dependencies = [ [[package]] name = "reth-rpc-layer" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-rpc-types-engine", "http", @@ -10221,7 +10221,7 @@ dependencies = [ [[package]] name = "reth-rpc-server-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10251,7 +10251,7 @@ dependencies = [ [[package]] name = "reth-stages" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10313,7 +10313,7 @@ dependencies = [ [[package]] name = "reth-stages-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10347,7 +10347,7 @@ dependencies = [ [[package]] name = "reth-stages-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "arbitrary", @@ -10363,7 +10363,7 @@ dependencies = [ [[package]] name = "reth-static-file" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "assert_matches", @@ -10386,7 +10386,7 @@ dependencies = [ [[package]] name = "reth-static-file-types" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "clap", @@ -10404,7 +10404,7 @@ dependencies = [ [[package]] name = "reth-storage-api" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eip7928", @@ -10431,7 +10431,7 @@ dependencies = [ [[package]] name = "reth-storage-errors" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-eips", "alloy-primitives", @@ -10447,7 +10447,7 @@ dependencies = [ [[package]] name = "reth-storage-overlay" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10476,7 +10476,7 @@ dependencies = [ [[package]] name = "reth-storage-rpc-provider" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10505,7 +10505,7 @@ dependencies = [ [[package]] name = "reth-tasks" -version = "2.5.1" +version = "2.5.2" dependencies = [ "crossbeam-utils", "dashmap", @@ -10525,7 +10525,7 @@ dependencies = [ [[package]] name = "reth-testing-utils" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10541,7 +10541,7 @@ dependencies = [ [[package]] name = "reth-tokio-util" -version = "2.5.1" +version = "2.5.2" dependencies = [ "tokio", "tokio-stream", @@ -10550,7 +10550,7 @@ dependencies = [ [[package]] name = "reth-tracing" -version = "2.5.1" +version = "2.5.2" dependencies = [ "clap", "eyre", @@ -10569,7 +10569,7 @@ dependencies = [ [[package]] name = "reth-tracing-otlp" -version = "2.5.1" +version = "2.5.2" dependencies = [ "base64 0.22.1", "clap", @@ -10587,7 +10587,7 @@ dependencies = [ [[package]] name = "reth-transaction-pool" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10638,7 +10638,7 @@ dependencies = [ [[package]] name = "reth-trie" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10670,7 +10670,7 @@ dependencies = [ [[package]] name = "reth-trie-common" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-eips", @@ -10702,7 +10702,7 @@ dependencies = [ [[package]] name = "reth-trie-db" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-consensus", "alloy-primitives", @@ -10729,7 +10729,7 @@ dependencies = [ [[package]] name = "reth-trie-parallel" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-evm", "alloy-primitives", @@ -10757,7 +10757,7 @@ dependencies = [ [[package]] name = "reth-trie-sparse" -version = "2.5.1" +version = "2.5.2" dependencies = [ "alloy-primitives", "alloy-rlp", diff --git a/Cargo.toml b/Cargo.toml index a54d01ac3fd..16277f885a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,5 @@ [workspace.package] -version = "2.5.1" +version = "2.5.2" edition = "2024" rust-version = "1.95" license = "MIT OR Apache-2.0" @@ -432,19 +432,27 @@ reth-zstd-compressors = { version = "0.6.0", default-features = false } # revm revm = { version = "42.0.1", default-features = false } -revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } +revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = [ + "llvm-prefer-static", +] } revm-inspectors = "0.42.0" # eth alloy-dyn-abi = "1.6.1" -alloy-primitives = { version = "1.6.1", default-features = false, features = ["map-foldhash"] } +alloy-primitives = { version = "1.6.1", default-features = false, features = [ + "map-foldhash", +] } alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } -alloy-eip7928 = { version = "0.4.5", default-features = false, features = ["rlp"] } +alloy-eip7928 = { version = "0.4.5", default-features = false, features = [ + "rlp", +] } alloy-evm = { version = "0.38.0", default-features = false } -alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } +alloy-rlp = { version = "0.3.16", default-features = false, features = [ + "core-net", +] } alloy-trie = { version = "0.9.4", default-features = false } alloy-hardforks = "0.4.7" @@ -457,10 +465,15 @@ alloy-json-rpc = { version = "2.3.0", default-features = false } alloy-network = { version = "2.3.0", default-features = false } alloy-network-primitives = { version = "2.3.0", default-features = false } alloy-node-bindings = "2.3.0" -alloy-provider = { version = "2.3.0", features = ["reqwest", "debug-api"], default-features = false } +alloy-provider = { version = "2.3.0", features = [ + "reqwest", + "debug-api", +], default-features = false } alloy-pubsub = { version = "2.3.0", default-features = false } alloy-rpc-client = { version = "2.3.0", default-features = false } -alloy-rpc-types = { version = "2.3.0", features = ["eth"], default-features = false } +alloy-rpc-types = { version = "2.3.0", features = [ + "eth", +], default-features = false } alloy-rpc-types-admin = { version = "2.3.0", default-features = false } alloy-rpc-types-anvil = { version = "2.3.0", default-features = false } alloy-rpc-types-beacon = { version = "2.3.0", default-features = false } @@ -474,7 +487,9 @@ alloy-serde = { version = "2.3.0", default-features = false } alloy-signer = { version = "2.3.0", default-features = false } alloy-signer-local = { version = "2.3.0", default-features = false } alloy-transport = { version = "2.3.0" } -alloy-transport-http = { version = "2.3.0", features = ["reqwest-rustls-tls"], default-features = false } +alloy-transport-http = { version = "2.3.0", features = [ + "reqwest-rustls-tls", +], default-features = false } alloy-transport-ipc = { version = "2.3.0", default-features = false } alloy-transport-ws = { version = "2.3.0", default-features = false } @@ -483,7 +498,10 @@ either = { version = "1.16.0", default-features = false } arrayvec = { version = "0.7.6", default-features = false } aquamarine = "0.6" auto_impl = "1" -backon = { version = "1.2", default-features = false, features = ["std-blocking-sleep", "tokio-sleep"] } +backon = { version = "1.2", default-features = false, features = [ + "std-blocking-sleep", + "tokio-sleep", +] } bincode = "1.3" bitflags = "2.4" boyer-moore-magiclen = "0.2.16" @@ -508,9 +526,13 @@ linked_hash_set = "0.1" libc = "0.2" lz4 = "1.28.1" modular-bitfield = "0.13.1" -notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] } +notify = { version = "8.0.0", default-features = false, features = [ + "macos_fsevent", +] } nybbles = { version = "0.4.8", default-features = false } -once_cell = { version = "1.19", default-features = false, features = ["critical-section"] } +once_cell = { version = "1.19", default-features = false, features = [ + "critical-section", +] } parking_lot = "0.12" quanta = "0.12" paste = "1.0" @@ -531,12 +553,16 @@ strum_macros = "0.27" syn = "2.0" thiserror = { version = "2.0.0", default-features = false } tar = "0.4.46" -tracing = { version = "0.1.0", default-features = false, features = ["attributes"] } +tracing = { version = "0.1.0", default-features = false, features = [ + "attributes", +] } tracing-appender = "0.2.5" url = { version = "2.3", default-features = false } zstd = "0.13" byteorder = "1" -fixed-cache = { version = "0.1.10", default-features = false, features = ["alloc"] } +fixed-cache = { version = "0.1.10", default-features = false, features = [ + "alloc", +] } moka = "0.12" tar-no-std = { version = "0.4.2", default-features = false } miniz_oxide = { version = "0.9.0", default-features = false } @@ -569,7 +595,10 @@ futures-util = { version = "0.3", default-features = false } hyper = "1.9" hyper-util = "0.1.5" pin-project = "1.1.12" -reqwest = { version = "0.13", default-features = false, features = ["rustls", "stream"] } +reqwest = { version = "0.13", default-features = false, features = [ + "rustls", + "stream", +] } tracing-futures = "0.2" tower = "0.5" tower-http = "0.6" @@ -594,7 +623,10 @@ proptest-arbitrary-interop = "0.1.0" # crypto enr = { version = "0.13", default-features = false } k256 = { version = "0.13", default-features = false, features = ["ecdsa"] } -secp256k1 = { version = "0.30", default-features = false, features = ["global-context", "recovery"] } +secp256k1 = { version = "0.30", default-features = false, features = [ + "global-context", + "recovery", +] } # rand 8 for secp256k1 rand_08 = { package = "rand", version = "0.8" } diff --git a/docs/vocs/vocs.config.ts b/docs/vocs/vocs.config.ts index 941f6e83ed6..6aa891ac973 100644 --- a/docs/vocs/vocs.config.ts +++ b/docs/vocs/vocs.config.ts @@ -23,7 +23,7 @@ export default defineConfig({ { text: 'Rustdocs', link: '/docs' }, { text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' }, { - text: 'v2.5.1', + text: 'v2.5.2', items: [ { text: 'Releases', From 5a6940e351fed80458fe6c9da8581cbe4b8bd036 Mon Sep 17 00:00:00 2001 From: Emma Jamieson-Hoare Date: Wed, 2 Sep 2026 11:35:39 +0100 Subject: [PATCH 135/148] chore: formatting --- Cargo.toml | 60 +++++++++++++----------------------------------------- 1 file changed, 14 insertions(+), 46 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 16277f885a0..d6d61f1b3e4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -432,27 +432,19 @@ reth-zstd-compressors = { version = "0.6.0", default-features = false } # revm revm = { version = "42.0.1", default-features = false } -revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = [ - "llvm-prefer-static", -] } +revmc = { git = "https://github.com/paradigmxyz/revmc", branch = "main", default-features = false, features = ["llvm-prefer-static"] } revm-inspectors = "0.42.0" # eth alloy-dyn-abi = "1.6.1" -alloy-primitives = { version = "1.6.1", default-features = false, features = [ - "map-foldhash", -] } +alloy-primitives = { version = "1.6.1", default-features = false, features = ["map-foldhash"] } alloy-sol-types = { version = "1.6.1", default-features = false } alloy-chains = { version = "0.2.33", default-features = false } alloy-eip2124 = { version = "0.2.0", default-features = false } -alloy-eip7928 = { version = "0.4.5", default-features = false, features = [ - "rlp", -] } +alloy-eip7928 = { version = "0.4.5", default-features = false, features = ["rlp"] } alloy-evm = { version = "0.38.0", default-features = false } -alloy-rlp = { version = "0.3.16", default-features = false, features = [ - "core-net", -] } +alloy-rlp = { version = "0.3.16", default-features = false, features = ["core-net"] } alloy-trie = { version = "0.9.4", default-features = false } alloy-hardforks = "0.4.7" @@ -465,15 +457,10 @@ alloy-json-rpc = { version = "2.3.0", default-features = false } alloy-network = { version = "2.3.0", default-features = false } alloy-network-primitives = { version = "2.3.0", default-features = false } alloy-node-bindings = "2.3.0" -alloy-provider = { version = "2.3.0", features = [ - "reqwest", - "debug-api", -], default-features = false } +alloy-provider = { version = "2.3.0", features = ["reqwest", "debug-api"], default-features = false } alloy-pubsub = { version = "2.3.0", default-features = false } alloy-rpc-client = { version = "2.3.0", default-features = false } -alloy-rpc-types = { version = "2.3.0", features = [ - "eth", -], default-features = false } +alloy-rpc-types = { version = "2.3.0", features = ["eth"], default-features = false } alloy-rpc-types-admin = { version = "2.3.0", default-features = false } alloy-rpc-types-anvil = { version = "2.3.0", default-features = false } alloy-rpc-types-beacon = { version = "2.3.0", default-features = false } @@ -487,9 +474,7 @@ alloy-serde = { version = "2.3.0", default-features = false } alloy-signer = { version = "2.3.0", default-features = false } alloy-signer-local = { version = "2.3.0", default-features = false } alloy-transport = { version = "2.3.0" } -alloy-transport-http = { version = "2.3.0", features = [ - "reqwest-rustls-tls", -], default-features = false } +alloy-transport-http = { version = "2.3.0", features = ["reqwest-rustls-tls"], default-features = false } alloy-transport-ipc = { version = "2.3.0", default-features = false } alloy-transport-ws = { version = "2.3.0", default-features = false } @@ -498,10 +483,7 @@ either = { version = "1.16.0", default-features = false } arrayvec = { version = "0.7.6", default-features = false } aquamarine = "0.6" auto_impl = "1" -backon = { version = "1.2", default-features = false, features = [ - "std-blocking-sleep", - "tokio-sleep", -] } +backon = { version = "1.2", default-features = false, features = ["std-blocking-sleep", "tokio-sleep"] } bincode = "1.3" bitflags = "2.4" boyer-moore-magiclen = "0.2.16" @@ -526,13 +508,9 @@ linked_hash_set = "0.1" libc = "0.2" lz4 = "1.28.1" modular-bitfield = "0.13.1" -notify = { version = "8.0.0", default-features = false, features = [ - "macos_fsevent", -] } +notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] } nybbles = { version = "0.4.8", default-features = false } -once_cell = { version = "1.19", default-features = false, features = [ - "critical-section", -] } +once_cell = { version = "1.19", default-features = false, features = ["critical-section"] } parking_lot = "0.12" quanta = "0.12" paste = "1.0" @@ -553,16 +531,12 @@ strum_macros = "0.27" syn = "2.0" thiserror = { version = "2.0.0", default-features = false } tar = "0.4.46" -tracing = { version = "0.1.0", default-features = false, features = [ - "attributes", -] } +tracing = { version = "0.1.0", default-features = false, features = ["attributes"] } tracing-appender = "0.2.5" url = { version = "2.3", default-features = false } zstd = "0.13" byteorder = "1" -fixed-cache = { version = "0.1.10", default-features = false, features = [ - "alloc", -] } +fixed-cache = { version = "0.1.10", default-features = false, features = ["alloc"] } moka = "0.12" tar-no-std = { version = "0.4.2", default-features = false } miniz_oxide = { version = "0.9.0", default-features = false } @@ -595,10 +569,7 @@ futures-util = { version = "0.3", default-features = false } hyper = "1.9" hyper-util = "0.1.5" pin-project = "1.1.12" -reqwest = { version = "0.13", default-features = false, features = [ - "rustls", - "stream", -] } +reqwest = { version = "0.13", default-features = false, features = ["rustls", "stream"] } tracing-futures = "0.2" tower = "0.5" tower-http = "0.6" @@ -623,10 +594,7 @@ proptest-arbitrary-interop = "0.1.0" # crypto enr = { version = "0.13", default-features = false } k256 = { version = "0.13", default-features = false, features = ["ecdsa"] } -secp256k1 = { version = "0.30", default-features = false, features = [ - "global-context", - "recovery", -] } +secp256k1 = { version = "0.30", default-features = false, features = ["global-context", "recovery"] } # rand 8 for secp256k1 rand_08 = { package = "rand", version = "0.8" } From 0b065f36e78ed87255f7b4ef0ad8c4a17d7d38f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Wed, 9 Sep 2026 12:10:20 -0400 Subject: [PATCH 136/148] Add the op-reth-v2.4.2-fh3.2 changelog section --- CHANGELOG.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e22ed39e5c6..172cf799f27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,20 @@ This changelog covers Firehose-specific changes only. For upstream reth changes, The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## op-reth-v2.4.2-fh3.2 + +### Fixed + +- Stop advertising a finalized block that is not an ancestor of the block being emitted. Every + `FIRE BLOCK` line carried the node's finalized head as of the moment the block executed, so a + block from a side branch was published with a LIB number naming the canonical chain's block at + that height; the consumer marked its own block at that height irreversible and then saw the + reorg replace it. The advertised block is now the node's finalized head when that head is on + the emitted block's own chain, the fork point where its branch left the canonical chain + otherwise, and genesis when the branch cannot be tied to the canonical chain at all. Seen on + BSC mainnet at block 120653740, where a four-block side branch was published with LIB + 120653741 — one of those blocks naming a height above its own number. + ## v2.3.0-fh-7 ### Fixed From 0eff4827c27c8489fb1d39610ff3c8c06ee20495 Mon Sep 17 00:00:00 2001 From: Matthias Seitz Date: Fri, 14 Aug 2026 15:57:51 +0000 Subject: [PATCH 137/148] fix(engine): keep messages responsive during pending handoff (#26708) (cherry picked from commit 90ea46d0665408804e5fa24c6f05e06c09b232bb) --- crates/engine/tree/src/tree/mod.rs | 23 ++++++++++------ crates/engine/tree/src/tree/tests.rs | 39 ++++++++++------------------ 2 files changed, 29 insertions(+), 33 deletions(-) diff --git a/crates/engine/tree/src/tree/mod.rs b/crates/engine/tree/src/tree/mod.rs index f0bb957fb3f..2995071a77e 100644 --- a/crates/engine/tree/src/tree/mod.rs +++ b/crates/engine/tree/src/tree/mod.rs @@ -710,17 +710,24 @@ where /// Blocks until the next event that can safely be processed is ready. /// - /// A pending persisted handoff only accepts the notification that all active payload builds - /// have finished. This keeps queued engine messages from starting replacement payload jobs - /// before the handoff can reclaim the in-memory overlay. Otherwise, uses biased selection to - /// prioritize persistence completion to update in-memory state and unblock further writes. + /// A pending persisted handoff keeps processing engine messages while prioritizing the + /// notification that all active payload builds have finished. This keeps the engine responsive + /// without reclaiming the in-memory overlay while a payload job may still access it. Otherwise, + /// uses biased selection to prioritize persistence completion to update in-memory state and + /// unblock further writes. fn wait_for_event(&mut self) -> LoopEvent { if self.pending_persisted_handoff.is_some() { self.metrics.engine.backpressure_active.set(0.0); - return match self.payload_build_finished.recv() { - Ok(()) => LoopEvent::PayloadBuildFinished, - Err(_) => LoopEvent::Disconnected, - }; + return crossbeam_channel::select_biased! { + recv(self.payload_build_finished) -> result => match result { + Ok(()) => LoopEvent::PayloadBuildFinished, + Err(_) => LoopEvent::Disconnected, + }, + recv(self.incoming) -> msg => match msg { + Ok(m) => LoopEvent::EngineMessage(m), + Err(_) => LoopEvent::Disconnected, + }, + } } // Take ownership of persistence rx if present diff --git a/crates/engine/tree/src/tree/tests.rs b/crates/engine/tree/src/tree/tests.rs index 53cbf1d4e5f..c2e517db218 100644 --- a/crates/engine/tree/src/tree/tests.rs +++ b/crates/engine/tree/src/tree/tests.rs @@ -709,7 +709,7 @@ fn persistence_handoff_waits_for_active_payload_jobs() { } #[test] -fn pending_persisted_handoff_blocks_engine_messages_until_payload_build_finishes() { +fn pending_persisted_handoff_keeps_engine_messages_responsive() { let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..4).collect(); let mut test_harness = TestHarness::with_config( MAINNET.clone(), @@ -732,34 +732,23 @@ fn pending_persisted_handoff_blocks_engine_messages_until_payload_build_finishes .unwrap(); assert!(test_harness.tree.pending_persisted_handoff.is_some()); - // Forkchoice updates received while the handoff is pending could start replacement payload - // builds and keep the final completion notification from reaching the tree. Queue a generic - // engine message to prove the gate leaves every message upstream until then. + // Engine messages remain processable while the handoff waits for the active payload build. test_harness.to_tree_tx.send(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)).unwrap(); + assert_matches!( + test_harness.tree.wait_for_event(), + super::LoopEvent::EngineMessage(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)) + ); + assert!(test_harness.tree.pending_persisted_handoff.is_some()); + assert!(test_harness.tree.state.tree_state.contains_hash(&persisted.hash)); - let (started_tx, started_rx) = std::sync::mpsc::sync_channel(0); - let (event_tx, event_rx) = std::sync::mpsc::sync_channel(1); - std::thread::scope(|scope| { - let tree = &mut test_harness.tree; - scope.spawn(move || { - started_tx.send(()).unwrap(); - event_tx - .send(matches!(tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished)) - .unwrap(); - }); - - started_rx.recv().unwrap(); - assert!( - event_rx.recv_timeout(Duration::from_millis(100)).is_err(), - "a pending handoff must prevent queued engine messages from being processed" - ); - - drop(payload_build); - assert_eq!(event_rx.recv_timeout(Duration::from_secs(1)), Ok(true)); - }); - + // Once the final lease drops, its completion is prioritized over queued engine messages so the + // handoff gets the first opportunity to reclaim the overlay. + test_harness.to_tree_tx.send(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)).unwrap(); + drop(payload_build); + assert_matches!(test_harness.tree.wait_for_event(), super::LoopEvent::PayloadBuildFinished); test_harness.tree.on_payload_build_finished().unwrap(); assert!(test_harness.tree.pending_persisted_handoff.is_none()); + assert!(!test_harness.tree.state.tree_state.contains_hash(&persisted.hash)); assert_matches!( test_harness.tree.wait_for_event(), super::LoopEvent::EngineMessage(FromEngine::Event(FromOrchestrator::BackfillSyncStarted)) From 0fbe428518594611bdd3fdd822eb5968823c8e66 Mon Sep 17 00:00:00 2001 From: Sebastian Stammler Date: Tue, 15 Sep 2026 15:59:19 +0200 Subject: [PATCH 138/148] ci: run fork validation through CircleCI act Squash the fork CI support from 04e73b7c2bdc8dcf3c1e8961c8d04b861bc15a8e and push-event support from b0d194cad81ddf9dcb954ecee95b22cc2d68b70b into one replayable maintenance commit. Preserve release action pins and advisory policy when resolving the replay. Select the book build job without entering Pages deployment. Refresh the standalone lockfile to patched h2 and rustls releases instead of suppressing RUSTSEC-2026-0258 and RUSTSEC-2026-0285; rustls requires the updated AWS-LC backend. Co-authored-by: Josh Klopfenstein Co-authored-by: falcorocks <14293929+falcorocks@users.noreply.github.com> Maintenance CI: Compare optimism push vectors against v2.5.2, retaining actual PR and merge-queue bases.: --- .circleci/config.yml | 209 ++++++++++++++++++++++++++++++ .circleci/prepare-act-event.sh | 102 +++++++++++++++ .github/scripts/check_rv32imac.sh | 2 +- .github/scripts/check_wasm.sh | 2 +- .github/workflows/book.yml | 2 + .github/workflows/compact.yml | 10 +- .github/workflows/e2e.yml | 4 + .github/workflows/integration.yml | 4 + .github/workflows/lint.yml | 20 ++- .github/workflows/stage.yml | 2 + .github/workflows/unit.yml | 6 + Cargo.lock | 21 +-- 12 files changed, 368 insertions(+), 16 deletions(-) create mode 100644 .circleci/config.yml create mode 100755 .circleci/prepare-act-event.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000000..d308228d379 --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,209 @@ +version: 2.1 + +# Runs this fork's validation workflows through pinned act because GitHub Actions is disabled. +# Pull requests and branch pushes run the PR checks; merge queues add stage.yml. Event metadata is +# generated once per CircleCI revision and shared by all shards. +# +# The read-only GitHub token is exposed to PR code, so its CircleCI context must remain +# restricted to trusted contributors. Keep CircleCI auto-cancellation enabled. +# +# The act binary, checksum, and runner image are pinned. Compiler caches are no-ops because +# act's local cache is not reusable across these machines; Pages plumbing is skipped after the +# validation build. Lint and unit jobs are distributed; other workflows run up to four jobs. +# +# Excluded: pr-title.yml, label-pr.yml, and the privileged cyclops-audit.yml integration. + +jobs: + prepare-act-event: + machine: + image: ubuntu-2404:2025.09.1 + resource_class: medium + steps: + - checkout + - run: + name: Snapshot GitHub event + command: | + mkdir -p /tmp/act-event + if ! .circleci/prepare-act-event.sh; then + echo "GitHub event preparation failed; act shards will report the failure" | + tee /tmp/act-event/failed + fi + - persist_to_workspace: + root: /tmp/act-event + paths: + - . + + act: + parameters: + wf: + type: string + job: + type: string + default: "" + concurrent_jobs: + type: integer + default: 4 + machine: + image: ubuntu-2404:2025.09.1 + resource_class: xlarge + steps: + - checkout + - attach_workspace: + at: /tmp/act-event + - run: + name: Check event snapshot + command: | + if [ -f /tmp/act-event/failed ]; then + cat /tmp/act-event/failed >&2 + exit 1 + fi + - run: + name: Install pinned act + command: | + set -euo pipefail + readonly ACT_VERSION="0.2.89-action-cache-fix.1" + readonly ACT_RELEASE="v0.2.89-action-cache-fix.4" + readonly ACT_BINARY="act_Linux_x86_64" + readonly ACT_SHA256="28bfcf1502f59a834b14f2fb23c39e5b253c2234d875592b5aa0a175f9674dce" + curl --fail --silent --show-error --location \ + "https://github.com/ethereum-optimism/act/releases/download/${ACT_RELEASE}/${ACT_BINARY}" \ + --output "/tmp/${ACT_BINARY}" + echo "${ACT_SHA256} /tmp/${ACT_BINARY}" | sha256sum --check + sudo install -m 0755 "/tmp/${ACT_BINARY}" /usr/local/bin/act + test "$(act --version)" = "act version ${ACT_VERSION}" + - run: + name: Stub unsupported non-check actions + command: | + mkdir -p /tmp/noop-action + printf '%s\n' \ + 'name: noop' \ + 'description: disabled service plumbing under act' \ + 'inputs:' \ + ' cache-on-failure: {required: false}' \ + ' cache-targets: {required: false}' \ + 'runs:' \ + ' using: composite' \ + ' steps:' \ + ' - run: echo "Service plumbing disabled under act"' \ + ' shell: bash' \ + > /tmp/noop-action/action.yml + - run: + name: Run << parameters.wf >> via act + no_output_timeout: 45m + command: | + set -euo pipefail + event_path=/tmp/act-event/github-event.json + event_name="$(jq -er 'if .pull_request then "pull_request" elif .merge_group then "merge_group" elif .ref and .after then "push" else error("unsupported event") end' "$event_path")" + base_ref="$(jq -r '(.pull_request.base.ref // .merge_group.base_ref // "") | sub("^refs/heads/"; "")' "$event_path")" + target_job="<< parameters.job >>" + # Validate documentation without entering Pages deployment on branch pushes. + if [ "<< parameters.wf >>" = "book.yml" ]; then + target_job=build + fi + act_job=() + if [ -n "$target_job" ]; then + case "$target_job" in + crate-checks-*) + partition="${target_job#crate-checks-}" + if [[ ! "$partition" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid crate-checks partition: $partition" >&2 + exit 1 + fi + act_job=(-j crate-checks --matrix "partition:${partition}") + ;; + *) + act_job=(-j "$target_job") + ;; + esac + fi + + act_env=(--env "RUSTC_WRAPPER=" --env "GITHUB_BASE_REF=${base_ref}") + if [ "<< parameters.wf >>" = "unit.yml" ] && [ "$target_job" = "state" ]; then + # EF tests use Rayon internally, so unrestricted nextest processes multiply + # runnable threads and starve valid_blocks past its upstream timeout. + act_env+=(--env "NEXTEST_TEST_THREADS=2") + fi + if [ "$event_name" != "pull_request" ]; then + act_env+=( + --env "GITHUB_REF=refs/heads/${CIRCLE_BRANCH}" + --env "GITHUB_SHA=${CIRCLE_SHA1}" + ) + fi + + act "$event_name" -W ".github/workflows/<< parameters.wf >>" \ + "${act_job[@]}" \ + --eventpath "$event_path" \ + -P "ubuntu-latest=catthehacker/ubuntu@sha256:b839c14c4410998529ec18f951262bdf87a2b23bc1467304d07b491b9455e074" \ + --concurrent-jobs << parameters.concurrent_jobs >> \ + "${act_env[@]}" \ + -s GITHUB_TOKEN="${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \ + --local-repository "mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696=/tmp/noop-action" \ + --local-repository "Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4=/tmp/noop-action" \ + --local-repository "actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d=/tmp/noop-action" \ + --local-repository "actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9=/tmp/noop-action" + +workflows: + reth-act: + jobs: + - prepare-act-event: + context: + - circleci-repo-readonly-authenticated-github-token + - act: + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-<< matrix.wf >>" + requires: + - prepare-act-event + matrix: + parameters: + wf: + - book.yml + - compact.yml + - e2e.yml + - grafana.yml + - integration.yml + - lint-actions.yml + - stage.yml + - act: + wf: lint.yml + concurrent_jobs: 1 + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-lint-<< matrix.job >>" + requires: + - prepare-act-event + matrix: + parameters: + job: + - clippy-binaries + - clippy + - wasm + - riscv + - crate-checks-1 + - crate-checks-2 + - crate-checks-3 + - msrv + - docs + - fmt + - udeps + - book + - typos + - check-toml + - grafana + - no-test-deps + - feature-propagation + - deny + - act: + wf: unit.yml + concurrent_jobs: 1 + context: + - circleci-repo-readonly-authenticated-github-token + name: "act-unit-<< matrix.job >>" + requires: + - prepare-act-event + matrix: + parameters: + job: + - test + - state + - doc diff --git a/.circleci/prepare-act-event.sh b/.circleci/prepare-act-event.sh new file mode 100755 index 00000000000..99cccb27f3c --- /dev/null +++ b/.circleci/prepare-act-event.sh @@ -0,0 +1,102 @@ +#!/usr/bin/env bash +set -euo pipefail + +readonly OUTPUT_DIR="${ACT_EVENT_OUTPUT_DIR:-/tmp/act-event}" +readonly EVENT_PATH="${OUTPUT_DIR}/github-event.json" + +mkdir -p "$OUTPUT_DIR" +rm -f "$EVENT_PATH" + +: "${CIRCLE_SHA1:?missing CircleCI revision}" +: "${CIRCLE_PROJECT_USERNAME:?missing CircleCI project owner}" +: "${CIRCLE_PROJECT_REPONAME:?missing CircleCI project repository}" + +repository="${CIRCLE_PROJECT_USERNAME}/${CIRCLE_PROJECT_REPONAME}" + +if [[ "${CIRCLE_BRANCH:-}" == gh-readonly-queue/* ]]; then + event_name="merge_group" +elif [[ -n "${CIRCLE_PULL_REQUEST:-}" ]]; then + event_name="pull_request" +else + event_name="push" +fi + +case "$event_name" in + pull_request) + : "${CIRCLE_PULL_REQUEST:?pull-request trigger must provide CIRCLE_PULL_REQUEST}" + pr_number="${CIRCLE_PULL_REQUEST##*/}" + if [[ ! "$pr_number" =~ ^[1-9][0-9]*$ ]]; then + echo "invalid pull-request URL: ${CIRCLE_PULL_REQUEST}" >&2 + exit 1 + fi + + pull_request_path="${OUTPUT_DIR}/pull-request.json" + curl --fail --silent --show-error --location \ + -H "Accept: application/vnd.github+json" \ + -H "Authorization: Bearer ${MISE_GITHUB_TOKEN:?missing restricted read-only GitHub token}" \ + -H "X-GitHub-Api-Version: 2022-11-28" \ + "https://api.github.com/repos/${repository}/pulls/${pr_number}" > "$pull_request_path" + + event_head_sha="$(jq -er '.head.sha' "$pull_request_path")" + if [[ "$event_head_sha" != "$CIRCLE_SHA1" ]]; then + echo "PR head does not match CircleCI revision: ${CIRCLE_SHA1} != ${event_head_sha}" >&2 + exit 1 + fi + + jq '{ + action: "synchronize", + number: .number, + pull_request: ., + repository: .base.repo, + sender: .user + }' "$pull_request_path" > "$EVENT_PATH" + base_ref="$(jq -er '.base.ref | select(length > 0)' "$pull_request_path")" + head_ref="$(jq -er '.head.ref | select(length > 0)' "$pull_request_path")" + echo "act event: pull_request #${pr_number}, ${head_ref} -> ${base_ref}" + ;; + push) + : "${CIRCLE_BRANCH:?push trigger must provide CIRCLE_BRANCH}" + jq -n \ + --arg ref "refs/heads/${CIRCLE_BRANCH}" \ + --arg after "$CIRCLE_SHA1" \ + '{ref: $ref, after: $after}' > "$EVENT_PATH" + echo "act event: push ${CIRCLE_BRANCH} at ${CIRCLE_SHA1}" + ;; + merge_group) + : "${CIRCLE_BRANCH:?merge-queue trigger must provide CIRCLE_BRANCH}" + if [[ ! "$CIRCLE_BRANCH" =~ ^gh-readonly-queue/(.+)/pr-([1-9][0-9]*)-[^/]+$ ]]; then + echo "merge_group requires a gh-readonly-queue//pr-- branch, got: ${CIRCLE_BRANCH}" >&2 + exit 1 + fi + base_ref="${BASH_REMATCH[1]}" + + if ! base_sha="$(git rev-parse "${CIRCLE_SHA1}^1" 2>/dev/null)"; then + git fetch --no-tags --depth=2 origin "$CIRCLE_SHA1" + base_sha="$(git rev-parse "${CIRCLE_SHA1}^1")" + fi + + jq -n \ + --arg head_sha "$CIRCLE_SHA1" \ + --arg head_ref "refs/heads/${CIRCLE_BRANCH}" \ + --arg base_sha "$base_sha" \ + --arg base_ref "refs/heads/${base_ref}" \ + --arg repository "$repository" \ + --arg repository_name "$CIRCLE_PROJECT_REPONAME" \ + --arg repository_owner "$CIRCLE_PROJECT_USERNAME" \ + '{ + action: "checks_requested", + merge_group: { + head_sha: $head_sha, + head_ref: $head_ref, + base_sha: $base_sha, + base_ref: $base_ref + }, + repository: { + full_name: $repository, + name: $repository_name, + owner: {login: $repository_owner} + } + }' > "$EVENT_PATH" + echo "act event: merge_group ${CIRCLE_BRANCH} -> ${base_ref} at ${base_sha}" + ;; +esac diff --git a/.github/scripts/check_rv32imac.sh b/.github/scripts/check_rv32imac.sh index 4e477cdbed1..988a95a3309 100755 --- a/.github/scripts/check_rv32imac.sh +++ b/.github/scripts/check_rv32imac.sh @@ -32,7 +32,7 @@ trap 'rm -rf -- "$tmpdir"' EXIT INT TERM for crate in "${crates_to_check[@]}"; do outfile="$tmpdir/$crate.log" - if cargo +stable build -p "$crate" --target riscv32imac-unknown-none-elf --no-default-features --color never >"$outfile" 2>&1; then + if cargo build -p "$crate" --target riscv32imac-unknown-none-elf --no-default-features --color never >"$outfile" 2>&1; then echo "✅ $crate" else echo "❌ $crate" diff --git a/.github/scripts/check_wasm.sh b/.github/scripts/check_wasm.sh index 13bea47cfea..376b1fac7cf 100755 --- a/.github/scripts/check_wasm.sh +++ b/.github/scripts/check_wasm.sh @@ -109,7 +109,7 @@ for crate in "${crates[@]}"; do fi outfile="$tmpdir/$crate.log" - if cargo +stable build -p "$crate" --target wasm32-wasip1 --no-default-features --color never >"$outfile" 2>&1; then + if cargo build -p "$crate" --target wasm32-wasip1 --no-default-features --color never >"$outfile" 2>&1; then echo "✅ $crate" else echo "❌ $crate" diff --git a/.github/workflows/book.yml b/.github/workflows/book.yml index c8cea3e8832..707ff64874e 100644 --- a/.github/workflows/book.yml +++ b/.github/workflows/book.yml @@ -40,6 +40,8 @@ jobs: - name: Install Rust nightly uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-14 - name: Build docs run: cd docs/vocs && bash scripts/build-cargo-docs.sh diff --git a/.github/workflows/compact.yml b/.github/workflows/compact.yml index 86f2592881a..80014d3793b 100644 --- a/.github/workflows/compact.yml +++ b/.github/workflows/compact.yml @@ -1,6 +1,6 @@ # Ensures that `Compact` codec changes are backwards compatible. # -# 1) checkout `main` +# 1) checkout the PR base, or the selected release for maintenance-branch pushes # 2) randomly generate and serialize to disk many different type vectors with `Compact` (eg. Header, Transaction, etc) # 3) checkout `pr` # 4) deserialize previously generated test vectors @@ -35,6 +35,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -42,9 +44,11 @@ jobs: - name: Checkout base uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - ref: ${{ github.base_ref || 'main' }} + # Use v2.5.2's commit: the fork carries its history but does not mirror upstream tags. + # The maintenance branch must decode its release's data, not unreleased main formats. + ref: ${{ github.base_ref || (github.ref == 'refs/heads/optimism' && '5a6940e351fed80458fe6c9da8581cbe4b8bd036' || 'main') }} persist-credentials: false - # On `main` branch, generates test vectors and serializes them to disk using `Compact`. + # On the comparison base, generate test vectors and serialize them to disk using `Compact`. - name: Generate compact vectors run: | ${{ matrix.bin }} -- test-vectors compact --write diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1abce78da42..bb1ca111517 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -35,6 +35,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: @@ -70,6 +72,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 591bc31304e..9dde9163659 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -42,6 +42,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - name: Install Geth run: .github/scripts/install_geth.sh - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 @@ -85,6 +87,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 562185f766b..59626fb4442 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -33,6 +33,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@clippy with: + toolchain: nightly-2026-07-14 components: clippy - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 @@ -61,6 +62,7 @@ jobs: - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly with: + toolchain: nightly-2026-07-14 components: clippy - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 @@ -80,8 +82,10 @@ jobs: with: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 + - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable with: + toolchain: "1.97.0" target: wasm32-wasip1 - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: @@ -93,7 +97,7 @@ jobs: - uses: dcarbone/install-jq-action@4fcb5062d7ce9bc4382d1a352d19ba3ba2c317c1 # v4.0.1 - name: Run Wasm checks run: | - sudo apt update && sudo apt install gcc-multilib + sudo apt update && sudo apt install --yes gcc-multilib .github/scripts/check_wasm.sh riscv: @@ -108,6 +112,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: dtolnay/rust-toolchain@stable with: + toolchain: "1.97.0" target: riscv32imac-unknown-none-elf - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: @@ -137,6 +142,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: cargo-hack @@ -182,6 +189,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -205,6 +214,7 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: dtolnay/rust-toolchain@nightly with: + toolchain: nightly-2026-07-14 components: rustfmt - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - name: Run fmt @@ -223,6 +233,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -245,6 +257,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@nightly + with: + toolchain: nightly-2026-07-14 - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -307,6 +321,8 @@ jobs: persist-credentials: false - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - name: Ensure no arbitrary or proptest dependency on default build run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0 @@ -322,6 +338,8 @@ jobs: persist-credentials: false - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - uses: taiki-e/cache-cargo-install-action@417450f3c33ee20393705369577571770643d4c7 # v3.0.7 diff --git a/.github/workflows/stage.yml b/.github/workflows/stage.yml index 8d0e2bf9d04..5b5d5d9eddf 100644 --- a/.github/workflows/stage.yml +++ b/.github/workflows/stage.yml @@ -39,6 +39,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: diff --git a/.github/workflows/unit.yml b/.github/workflows/unit.yml index 74986cbb3b9..465747462c3 100644 --- a/.github/workflows/unit.yml +++ b/.github/workflows/unit.yml @@ -42,6 +42,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: @@ -97,6 +99,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: taiki-e/install-action@41049aa56687c35e0afa74eed4f09cec4f9afabf # v2.85.2 with: tool: nextest @@ -121,6 +125,8 @@ jobs: - uses: rui314/setup-mold@9c9c13bf4c3f1adef0cc596abc155580bcb04444 # v1 - run: .github/scripts/install_llvm.sh ubuntu - uses: dtolnay/rust-toolchain@stable + with: + toolchain: "1.97.0" - uses: mozilla-actions/sccache-action@9e7fa8a12102821edf02ca5dbea1acd0f89a2696 # v0.0.10 - uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1 with: diff --git a/Cargo.lock b/Cargo.lock index 016f891a405..f33ed1c6760 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1573,9 +1573,9 @@ checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" [[package]] name = "aws-lc-rs" -version = "1.16.3" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" +checksum = "b281d307588d634de920874890732659e2e7672f72b5e10e81badc1a8a83621e" dependencies = [ "aws-lc-sys", "untrusted 0.7.1", @@ -1584,14 +1584,15 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.40.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" +checksum = "9bff6c3b54fad79a2e60b8102caf565819711497c1f5f092f49508e2f5c31b27" dependencies = [ "cc", "cmake", "dunce", "fs_extra", + "pkg-config", ] [[package]] @@ -4373,9 +4374,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.4.14" +version = "0.4.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +checksum = "a9f37a958b41b3b19ee2707c06439c0e9e547e847223eb791ecb0cb821c65e27" dependencies = [ "atomic-waker", "bytes", @@ -11377,9 +11378,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.38" +version = "0.23.45" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" +checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" dependencies = [ "aws-lc-rs", "log", @@ -11463,9 +11464,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.13" +version = "0.103.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" dependencies = [ "aws-lc-rs", "ring", From 4ca11e74f3b15c2ac77bec5da7dd5bfb95677fd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 17 Sep 2026 14:56:29 -0400 Subject: [PATCH 139/148] firehose: cap keccak preimages at 256 bytes Preimages larger than that come from a contract hashing its own data rather than deriving a storage slot, and are left out of the map instead of truncated: a truncated preimage no longer hashes back to its key. Matches the geth firehose tracer. --- crates/firehose/src/inspector.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/crates/firehose/src/inspector.rs b/crates/firehose/src/inspector.rs index a675ceb3e9f..5541bc5cbcc 100644 --- a/crates/firehose/src/inspector.rs +++ b/crates/firehose/src/inspector.rs @@ -19,6 +19,16 @@ use std::{ fmt::Debug, }; +/// The largest preimage, in bytes, recorded in `Call.keccak_preimages`. The map exists so a +/// consumer can walk a storage slot back to the expression that produced it, and Solidity's slot +/// derivations are all small: 32 bytes for a dynamic array or a long `bytes`/`string`, 64 bytes +/// for a mapping with a value-type key (one level per nesting), and 32 bytes plus the key for a +/// `mapping(string => V)` or `mapping(bytes => V)`. 256 bytes covers those with room for a +/// 224-byte dynamic key. Anything larger is contract-level hashing, not slot derivation, and is +/// dropped rather than truncated: a truncated preimage does not hash back to its key and would be +/// worse than no entry at all. +const MAX_KECCAK_PREIMAGE_SIZE: usize = 256; + struct StepContext { start_journal_idx: usize, opcode: u8, @@ -197,6 +207,10 @@ impl<'a> FirehoseInspector<'a> { }; let len = size.saturating_to::(); + if len > MAX_KECCAK_PREIMAGE_SIZE { + return None; + } + if len == 0 { return Some((alloy_primitives::utils::KECCAK256_EMPTY, Vec::new())); } @@ -213,12 +227,10 @@ impl<'a> FirehoseInspector<'a> { // Memory not yet resized (step fires before resize_memory!). // Zero-pad like Geth's Memory.GetPtr to produce a complete preimage. // - // `size` is an untrusted stack operand. Growing memory to cover it - // costs quadratic gas (3*w + w^2/512, w = words). Anything the - // remaining gas can't pay for OOG-reverts before the opcode runs, so - // step_end never emits this preimage. Cap the zero-pad allocation to - // that gas-affordable bound so a beefy machine doesn't allocate GBs - // for a keccak Ethereum would reject anyway (and never overflows). + // Growing memory to cover `size` costs quadratic gas (3*w + w^2/512, + // w = words). Anything the remaining gas can't pay for OOG-reverts + // before the opcode runs, so step_end never emits this preimage; skip + // it here as well rather than hashing a region that never existed. // // From 3*w + w^2/512 <= gas, a generous bound is w <= floor(sqrt(512*gas)) // (dropping the +3*w term only makes the cap larger, never rejecting a From 409e21bb48bf0f03fd25198b041eba9dd66aba64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Mon, 28 Sep 2026 15:24:14 -0400 Subject: [PATCH 140/148] Use firehose-tracer with the keccak preimage filter Patched to the evm-firehose-tracer-rs branch until firehose-tracer 5.5.0 is published. BlockData gained the EIP-7928 fields in 5.4.3; they stay unset on this chain. --- Cargo.lock | 27 +++++++++++++-------------- Cargo.toml | 5 +++++ crates/firehose/src/mapper.rs | 3 +++ 3 files changed, 21 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 7befcc4d543..f8cd432909e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1030,7 +1030,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1041,7 +1041,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -1907,7 +1907,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.23.0", + "darling 0.21.3", "ident_case", "prettyplease", "proc-macro2", @@ -3369,7 +3369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -3914,9 +3914,8 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "firehose-tracer" -version = "5.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "68617cc0fb3d0dfb49cbe29f3e05969fcabf93fcecaa1e0fda577c7c6c95494c" +version = "5.4.4" +source = "git+https://github.com/streamingfast/evm-firehose-tracer-rs.git?branch=feature%2Fkeccak-preimage-filter#de027964ce98a184ce3f17c8266184b780658126" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5152,7 +5151,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -6229,7 +6228,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -11309,7 +11308,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -11389,7 +11388,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -12011,7 +12010,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.60.2", ] [[package]] @@ -12259,7 +12258,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] @@ -13544,7 +13543,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 33fb6e4dd77..cd05a9335d1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -709,6 +709,11 @@ reth-firehose = { path = "crates/firehose" } reth-firehose-tests = { path = "crates/firehose-tests" } [patch.crates-io] +## Firehose additions +# Temporary: the keccak preimage filter is not on crates.io yet. Remove once firehose-tracer +# 5.5.0 is published and the lock points to it. +firehose-tracer = { git = "https://github.com/streamingfast/evm-firehose-tracer-rs.git", branch = "feature/keccak-preimage-filter" } + ## Firehose additions # SF fork of alloy-evm that routes system calls (EIP-4788, EIP-2935, etc.) through the Inspector. # See https://github.com/alloy-rs/evm/pull/323 for the upstream PR. diff --git a/crates/firehose/src/mapper.rs b/crates/firehose/src/mapper.rs index a27347920a7..b313de679f9 100644 --- a/crates/firehose/src/mapper.rs +++ b/crates/firehose/src/mapper.rs @@ -85,6 +85,9 @@ where tx_dependency: None, // EIP-7843: Amsterdam slot number — not yet exposed by reth Header trait slot_number: None, + // EIP-7928: Amsterdam block access list, not active on this chain + block_access_list_hash: None, + block_access_list_rlp: None, } } From 82c39f1b2bfe544494fb81cf62ca36930c4eab84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Mon, 28 Sep 2026 15:42:24 -0400 Subject: [PATCH 141/148] Add changelog entry for the keccak preimage filter --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 172cf799f27..cb00ce6ac27 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ## op-reth-v2.4.2-fh3.2 +### Changed + +- Firehose `Call.keccak_preimages` now keeps only the preimages that explain a storage change key of the transaction or system call; set `FIREHOSE_ETHEREUM_TRACER_DISABLE_KECCAK_FILTER=true` to keep them all (streamingfast/evm-firehose-tracer-rs#36). + ### Fixed - Stop advertising a finalized block that is not an ancestor of the block being emitted. Every From ec4a4f8b44928ac0a83600933191b686cdb615d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Tue, 29 Sep 2026 09:08:35 -0400 Subject: [PATCH 142/148] Pick up the always-on keccak preimage filter --- CHANGELOG.md | 2 +- Cargo.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb00ce6ac27..3e67921c5fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,7 +11,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed -- Firehose `Call.keccak_preimages` now keeps only the preimages that explain a storage change key of the transaction or system call; set `FIREHOSE_ETHEREUM_TRACER_DISABLE_KECCAK_FILTER=true` to keep them all (streamingfast/evm-firehose-tracer-rs#36). +- Firehose `Call.keccak_preimages` now keeps only the preimages that explain a storage change key of the transaction or system call (streamingfast/evm-firehose-tracer-rs#36). ### Fixed diff --git a/Cargo.lock b/Cargo.lock index f8cd432909e..39c2d5e6f8d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3369,7 +3369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3915,7 +3915,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "firehose-tracer" version = "5.4.4" -source = "git+https://github.com/streamingfast/evm-firehose-tracer-rs.git?branch=feature%2Fkeccak-preimage-filter#de027964ce98a184ce3f17c8266184b780658126" +source = "git+https://github.com/streamingfast/evm-firehose-tracer-rs.git?branch=feature%2Fkeccak-preimage-filter#6fa66194dfcc699f8b2b19d5e4912f3b03b3e191" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5151,7 +5151,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11308,7 +11308,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11388,7 +11388,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -12258,7 +12258,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -13543,7 +13543,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] From 638951521c45f1fbc1f286f8c3add9c066290973 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Tue, 29 Sep 2026 11:46:19 -0400 Subject: [PATCH 143/148] Point firehose-tracer at the merged keccak preimage filter --- Cargo.lock | 20 ++++++++++---------- Cargo.toml | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 39c2d5e6f8d..7a1f323d5e6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2409,7 +2409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3369,7 +3369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -3915,7 +3915,7 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "firehose-tracer" version = "5.4.4" -source = "git+https://github.com/streamingfast/evm-firehose-tracer-rs.git?branch=feature%2Fkeccak-preimage-filter#6fa66194dfcc699f8b2b19d5e4912f3b03b3e191" +source = "git+https://github.com/streamingfast/evm-firehose-tracer-rs.git?rev=3321aec1ebb9e215de2bd9cc58e7f1726496e47f#3321aec1ebb9e215de2bd9cc58e7f1726496e47f" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5151,7 +5151,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -6228,7 +6228,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -11308,7 +11308,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -11367,7 +11367,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 0.26.11", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -11388,7 +11388,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -12258,7 +12258,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] @@ -13543,7 +13543,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.52.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index cd05a9335d1..2111220281f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -712,7 +712,7 @@ reth-firehose-tests = { path = "crates/firehose-tests" } ## Firehose additions # Temporary: the keccak preimage filter is not on crates.io yet. Remove once firehose-tracer # 5.5.0 is published and the lock points to it. -firehose-tracer = { git = "https://github.com/streamingfast/evm-firehose-tracer-rs.git", branch = "feature/keccak-preimage-filter" } +firehose-tracer = { git = "https://github.com/streamingfast/evm-firehose-tracer-rs.git", rev = "3321aec1ebb9e215de2bd9cc58e7f1726496e47f" } ## Firehose additions # SF fork of alloy-evm that routes system calls (EIP-4788, EIP-2935, etc.) through the Inspector. From 28c0c1917f391147b1cec799b1ac0f7208e5630b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Tue, 29 Sep 2026 16:08:01 -0400 Subject: [PATCH 144/148] Use firehose-tracer 5.5.0 --- CHANGELOG.md | 1 + Cargo.lock | 31 ++++++++++++++++--------------- Cargo.toml | 7 +------ 3 files changed, 18 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3e67921c5fd..f4dbae69235 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed - Firehose `Call.keccak_preimages` now keeps only the preimages that explain a storage change key of the transaction or system call (streamingfast/evm-firehose-tracer-rs#36). +- Firehose limits the call input and return data a block records (firehose-tracer 5.5.0): past 50 MiB of internal call input or 25 MiB of return data in a transaction, later calls keep only their 4-byte selector or no return data, and set `Call.input_truncated` / `Call.return_data_truncated`. ### Fixed diff --git a/Cargo.lock b/Cargo.lock index 7a1f323d5e6..ade7745bd3f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1030,7 +1030,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1041,7 +1041,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -1907,7 +1907,7 @@ version = "3.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6dee98b0db6a962de883bf5d20362dee4d7ca0d12fe39a7c6c73c844e1cd7c1f" dependencies = [ - "darling 0.21.3", + "darling 0.23.0", "ident_case", "prettyplease", "proc-macro2", @@ -2409,7 +2409,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "117725a109d387c937a1533ce01b450cbde6b88abceea8473c4d7a85853cda3c" dependencies = [ "lazy_static", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -3369,7 +3369,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -3914,8 +3914,9 @@ checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" [[package]] name = "firehose-tracer" -version = "5.4.4" -source = "git+https://github.com/streamingfast/evm-firehose-tracer-rs.git?rev=3321aec1ebb9e215de2bd9cc58e7f1726496e47f#3321aec1ebb9e215de2bd9cc58e7f1726496e47f" +version = "5.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a53eb86c6d17bd4e5517e46f56498d33d128d0641d78d15b86cf2c0605c814ee" dependencies = [ "alloy-consensus", "alloy-eips", @@ -5151,7 +5152,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -6228,7 +6229,7 @@ version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -11308,7 +11309,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -11367,7 +11368,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 0.26.11", - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -11388,7 +11389,7 @@ dependencies = [ "security-framework", "security-framework-sys", "webpki-root-certs 1.0.7", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -12010,7 +12011,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "52d1cfed4120b4d927bf7c0f86d2087a4a7d6027c906d9f9d525a80573b9be51" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -12258,7 +12259,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix", - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] @@ -13543,7 +13544,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.61.2", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 2111220281f..1711f2d29c8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -704,16 +704,11 @@ vergen-git2 = "10.0.1" ipnet = "2.11" ## Firehose Additions -firehose-tracer = "5" +firehose-tracer = "5.5.0" reth-firehose = { path = "crates/firehose" } reth-firehose-tests = { path = "crates/firehose-tests" } [patch.crates-io] -## Firehose additions -# Temporary: the keccak preimage filter is not on crates.io yet. Remove once firehose-tracer -# 5.5.0 is published and the lock points to it. -firehose-tracer = { git = "https://github.com/streamingfast/evm-firehose-tracer-rs.git", rev = "3321aec1ebb9e215de2bd9cc58e7f1726496e47f" } - ## Firehose additions # SF fork of alloy-evm that routes system calls (EIP-4788, EIP-2935, etc.) through the Inspector. # See https://github.com/alloy-rs/evm/pull/323 for the upstream PR. From 77e6cc54d610cfbc63f6bbe927375185ee2c14fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Tue, 29 Sep 2026 16:23:38 -0400 Subject: [PATCH 145/148] Prepare op-reth-v2.4.2-fh3.1-3 release notes --- CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f4dbae69235..75285afae94 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,13 +7,15 @@ This changelog covers Firehose-specific changes only. For upstream reth changes, The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). -## op-reth-v2.4.2-fh3.2 +## op-reth-v2.4.2-fh3.1-3 ### Changed - Firehose `Call.keccak_preimages` now keeps only the preimages that explain a storage change key of the transaction or system call (streamingfast/evm-firehose-tracer-rs#36). - Firehose limits the call input and return data a block records (firehose-tracer 5.5.0): past 50 MiB of internal call input or 25 MiB of return data in a transaction, later calls keep only their 4-byte selector or no return data, and set `Call.input_truncated` / `Call.return_data_truncated`. +## op-reth-v2.4.2-fh3.1-2 + ### Fixed - Stop advertising a finalized block that is not an ancestor of the block being emitted. Every From 410e113ebe012db07910c2ae7e117dc7726575c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Tue, 29 Sep 2026 16:27:30 -0400 Subject: [PATCH 146/148] Reword the keccak filter and call limit entries --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75285afae94..2c13884b9c1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,8 +11,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed -- Firehose `Call.keccak_preimages` now keeps only the preimages that explain a storage change key of the transaction or system call (streamingfast/evm-firehose-tracer-rs#36). -- Firehose limits the call input and return data a block records (firehose-tracer 5.5.0): past 50 MiB of internal call input or 25 MiB of return data in a transaction, later calls keep only their 4-byte selector or no return data, and set `Call.input_truncated` / `Call.return_data_truncated`. +- Firehose `Call.keccak_preimages` now keeps only the preimages that explain a storage slot written by the transaction: the hash is the storage key, the storage key is the hash plus a small offset (a struct field or array position), or the hash appears inside another kept preimage (up to 16 levels). Preimages of hashes that never lead to a written storage slot, and preimages over 256 bytes, are no longer recorded (see streamingfast/reth#57). +- Firehose limits the call input and return data a transaction records: past 50 MiB of internal call input or 25 MiB of return data, later calls keep only the 4-byte selector of their input and no return data, and set `Call.input_truncated` / `Call.return_data_truncated`. The root call is never cut, and a block that would still encode to more than 1 GiB has both limits halved until it fits (see streamingfast/reth#57). ## op-reth-v2.4.2-fh3.1-2 From dd5409c26843fdacf366a70a1f028d2e62a8d2b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Thu, 22 Oct 2026 10:21:01 -0400 Subject: [PATCH 147/148] Add the op-reth-v2.6.1-fh3.1 changelog section --- CHANGELOG.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c13884b9c1..75e7227662e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,18 @@ This changelog covers Firehose-specific changes only. For upstream reth changes, The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). +## op-reth-v2.6.1-fh3.1 + +### Changed + +- Merged ethereum-optimism/reth `0fbe4285` (reth v2.5.2 plus two commits), the reth pin of op-reth + v2.6.1. The previous pin was op-rs/reth `aef8d3ef`, based on reth v2.4.0. +- The `alloy-evm` patch moves to `streamingfast/evm` `v0.38.0-sf` (revm 42), so block traces keep + their `systemCalls`. +- Firehose code changes take the previous code hash and code from the `CodeChange` journal entry, + which revm 42 now records, instead of assuming empty code. A contract creation starts from empty + code, so its code change is reported as before. + ## op-reth-v2.4.2-fh3.1-3 ### Changed From e2428f7e55b4ebe51f5002dd2bbd0c7fad1aca2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ste=CC=81phane=20Duchesneau?= Date: Fri, 2 Oct 2026 17:09:54 -0400 Subject: [PATCH 148/148] Emit Firehose init and genesis without an ExEx The Firehose ExEx only emitted FIRE INIT and the genesis block; its block tracing was disabled. Emit both from the component-initialized hook, which runs before the consensus engine exists, so no block is traced first. --- CHANGELOG.md | 10 + Cargo.lock | 3 - bin/reth/src/main.rs | 4 +- crates/firehose/Cargo.toml | 3 - crates/firehose/src/block_tracer.rs | 2 +- crates/firehose/src/init.rs | 77 +++++++ crates/firehose/src/lib.rs | 8 +- crates/firehose/src/mapper.rs | 8 +- crates/firehose/src/runner.rs | 332 ---------------------------- 9 files changed, 94 insertions(+), 353 deletions(-) create mode 100644 crates/firehose/src/init.rs delete mode 100644 crates/firehose/src/runner.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 75e7227662e..5febb92ccfe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). ### Changed +- `reth` no longer installs a Firehose execution extension (ExEx). It only emitted `FIRE INIT` + and the genesis block; blocks are traced by the engine. Both are now emitted at startup before + the consensus engine exists, so no block can be traced ahead of `FIRE INIT`, and the node + stops writing the ExEx write-ahead log. - Merged ethereum-optimism/reth `0fbe4285` (reth v2.5.2 plus two commits), the reth pin of op-reth v2.6.1. The previous pin was op-rs/reth `aef8d3ef`, based on reth v2.4.0. - The `alloy-evm` patch moves to `streamingfast/evm` `v0.38.0-sf` (revm 42), so block traces keep @@ -19,6 +23,12 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/). which revm 42 now records, instead of assuming empty code. A contract creation starts from empty code, so its code change is reported as before. +### Removed + +- `reth_firehose::run_exex` and `reth_firehose::emit_genesis_block_if_empty`. Call + `reth_firehose::init_blockchain(&provider)` from the node builder's `on_component_initialized` + hook instead. `reth-firehose` no longer depends on `reth-exex`. + ## op-reth-v2.4.2-fh3.1-3 ### Changed diff --git a/Cargo.lock b/Cargo.lock index 5e9e5ed1ea5..c532ac33c12 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9095,15 +9095,12 @@ dependencies = [ "base64 0.22.1", "eyre", "firehose-tracer", - "futures", "prost", "rayon", "reth-chainspec", - "reth-ethereum-forks", "reth-ethereum-primitives", "reth-evm", "reth-execution-types", - "reth-exex", "reth-node-api", "reth-primitives-traits", "reth-provider", diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 5080b023093..ad875c75dd8 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -50,9 +50,7 @@ fn main() { EthereumNode::components().executor(FirehoseExecutorBuilder::default()), ) .with_add_ons(EthereumAddOns::default()) - .install_exex("firehose", |ctx| async move { - Ok(async move { reth_firehose::run_exex(ctx).await }) - }) + .on_component_initialized(|node| reth_firehose::init_blockchain(&node.provider)) .launch_with_debug_capabilities() .await?; diff --git a/crates/firehose/Cargo.toml b/crates/firehose/Cargo.toml index 6daa86ce207..221f9d159e9 100644 --- a/crates/firehose/Cargo.toml +++ b/crates/firehose/Cargo.toml @@ -15,10 +15,8 @@ firehose-tracer.workspace = true # reth reth-chainspec.workspace = true -reth-ethereum-forks.workspace = true reth-evm.workspace = true reth-execution-types.workspace = true -reth-exex.workspace = true reth-node-api.workspace = true reth-ethereum-primitives.workspace = true reth-primitives-traits.workspace = true @@ -37,7 +35,6 @@ alloy-rlp.workspace = true # std eyre.workspace = true -futures.workspace = true rayon = { workspace = true, optional = true } [dev-dependencies] diff --git a/crates/firehose/src/block_tracer.rs b/crates/firehose/src/block_tracer.rs index b91114e4f5e..e92c9b037a6 100644 --- a/crates/firehose/src/block_tracer.rs +++ b/crates/firehose/src/block_tracer.rs @@ -54,7 +54,7 @@ impl FirehoseBlockTracer { /// /// The genesis block (block 0) never goes through this guard: reth writes it directly to /// the database without executing it, and it is emitted standalone via - /// [`crate::runner::emit_genesis_block_if_empty`] at startup. + /// [`crate::init::emit_genesis_block_on_empty_chain`] at startup. /// /// Takes a [`SealedBlock`] rather than a `RecoveredBlock` so the guard can be started before /// transaction senders have been recovered. Block-level data read by the mapper is signer-free. diff --git a/crates/firehose/src/init.rs b/crates/firehose/src/init.rs new file mode 100644 index 00000000000..588e5aa4e46 --- /dev/null +++ b/crates/firehose/src/init.rs @@ -0,0 +1,77 @@ +use crate::{mapper, prelude::*}; +use reth_chainspec::EthChainSpec; +use reth_provider::{BlockReader, ChainSpecProvider}; + +/// Emits `FIRE INIT` and, on an empty chain, the genesis block. +/// +/// Call it once from the node builder's `on_component_initialized` hook. The tracer panics on a +/// block that starts before `FIRE INIT`, and the consensus engine can execute blocks before the +/// `on_node_started` hook runs. +pub fn init_blockchain

(provider: &P) -> eyre::Result<()> +where + P: BlockReader + ChainSpecProvider, + ::Header: BlockHeader + alloy_primitives::Sealable, + <::Body as reth_primitives_traits::BlockBody>::OmmerHeader: + BlockHeader + alloy_primitives::Sealable, +{ + let chain_spec = provider.chain_spec(); + crate::tracer().on_blockchain_init( + "reth", + env!("CARGO_PKG_VERSION"), + firehose_tracer::config::ChainConfig::new(chain_spec.chain().id()), + ); + + emit_genesis_block_on_empty_chain(provider, chain_spec.genesis()) +} + +/// Emits the genesis block (block 0) to the Firehose stream if the chain is empty (head still +/// at genesis). +/// +/// Reth never executes the genesis block — it is written straight to the database during init — +/// so no execution hook ever fires for it. Without this, a Firehose stream started from scratch +/// begins at block 1 and downstream consumers waiting for block 0 stall forever. Mirrors geth's +/// `OnGenesisBlock` hook: emitted with the full genesis alloc, only when starting on an empty +/// chain. +/// +/// Call it once at node startup, after `on_blockchain_init` and before any block is traced. +/// Embedders that report their own node name or version in `FIRE INIT` call this directly instead +/// of [`init_blockchain`]. +pub fn emit_genesis_block_on_empty_chain

( + provider: &P, + genesis: &alloy_genesis::Genesis, +) -> eyre::Result<()> +where + P: BlockReader, + ::Header: BlockHeader + alloy_primitives::Sealable, + <::Body as reth_primitives_traits::BlockBody>::OmmerHeader: + BlockHeader + alloy_primitives::Sealable, +{ + use reth_primitives_traits::Block as _; + + // OP-stack chains may place their genesis at a non-zero height (e.g. OP mainnet's + // post-bedrock genesis); `genesis.number` carries it, defaulting to 0. + let genesis_number = genesis.number.unwrap_or(0); + if provider.best_block_number()? != genesis_number { + return Ok(()); + } + + let genesis_block = provider + .block(genesis_number.into())? + .ok_or_else(|| eyre::eyre!("genesis block {genesis_number} not found in database"))? + .seal_slow(); + + crate::tracer().on_genesis_block( + firehose_tracer::types::BlockEvent { + block: mapper::to_block_data(&genesis_block), + // Genesis is finalized by definition. + finalized: Some(firehose_tracer::types::FinalizedBlockRef { + number: genesis_number, + hash: Some(genesis_block.hash()), + }), + flash_block: None, + }, + mapper::to_genesis_alloc(genesis), + ); + + Ok(()) +} diff --git a/crates/firehose/src/lib.rs b/crates/firehose/src/lib.rs index 17e16859348..3ebe6b57bc9 100644 --- a/crates/firehose/src/lib.rs +++ b/crates/firehose/src/lib.rs @@ -1,6 +1,6 @@ //! Firehose crate providing blockchain data processing modules. //! -//! This crate contains modules for inspection, mapping, prelude utilities, and running tasks. +//! This crate contains modules for inspection, mapping, prelude utilities, and node startup. /// Block-level drop guard that manages the Firehose tracer lifecycle across validation. pub mod block_tracer; @@ -8,14 +8,14 @@ pub mod block_tracer; pub mod executor; /// Resolves which finalized block a Firehose block event may advertise. pub mod finality; +/// Startup module emitting `FIRE INIT` and the genesis block. +pub mod init; /// Inspector module for analyzing blockchain data. pub mod inspector; /// Mapper module for transforming blockchain data. pub mod mapper; /// Prelude module with common imports and utilities. pub mod prelude; -/// Runner module for executing processing tasks. -pub mod runner; pub use block_tracer::{FirehoseBlockTracer, GlobalTracerGuard}; pub use executor::{ @@ -25,7 +25,7 @@ pub use executor::{ }; pub use finality::finalized_ref_for_block; pub use firehose_tracer::types::FinalizedBlockRef; -pub use runner::{emit_genesis_block_if_empty, emit_genesis_block_on_empty_chain, run_exex}; +pub use init::{emit_genesis_block_on_empty_chain, init_blockchain}; use std::{ io::Write, diff --git a/crates/firehose/src/mapper.rs b/crates/firehose/src/mapper.rs index b313de679f9..3a5f8f3856c 100644 --- a/crates/firehose/src/mapper.rs +++ b/crates/firehose/src/mapper.rs @@ -15,7 +15,7 @@ use firehose_tracer::types::{ AccessTuple, BlockData, GenesisAlloc, SetCodeAuthorization, TxEvent, UncleData, WithdrawalData, }; use reth_primitives_traits::{Block as BlockTrait, BlockBody}; -use reth_provider::{AccountReader, ProviderResult}; +use reth_provider::AccountReader; pub(crate) fn to_genesis_alloc(genesis: &Genesis) -> GenesisAlloc { genesis @@ -123,12 +123,6 @@ where .collect() } -pub(crate) fn to_finalized_ref( - block_ref: ProviderResult>, -) -> Option { - finalized_ref_from_num_hash(block_ref.ok().flatten()) -} - /// Builds a Firehose [`FinalizedBlockRef`](firehose_tracer::types::FinalizedBlockRef) from the /// finalized block's number/hash, if known. /// diff --git a/crates/firehose/src/runner.rs b/crates/firehose/src/runner.rs deleted file mode 100644 index d43716c9a6a..00000000000 --- a/crates/firehose/src/runner.rs +++ /dev/null @@ -1,332 +0,0 @@ -use crate::{inspector, mapper, prelude::*}; -use alloy_consensus::transaction::TxHashRef as _; -use alloy_primitives::Bytes; -use eyre::Context; -use futures::StreamExt; -use reth_chainspec::EthChainSpec; -use reth_ethereum_forks::EthereumHardforks; -use reth_evm::execute::BlockExecutor; -use reth_exex::{ExExContext, ExExEvent}; -use reth_provider::{ - BlockIdReader, BlockNumReader, BlockReader, StateProviderBox, StateProviderFactory, -}; -use reth_revm::{ - database::StateProviderDatabase, - revm::{context::Block as _, Database as _}, - State, -}; - -/// Executes EVM transactions in a block one by one, firing tracer hooks at the appropriate times. -/// -/// This is a re-implementation for usage within ExEx and compatible with Firehose Geth Live -/// Tracing. -pub fn trace_block( - ctx: &ExExContext, - evm_config: &Node::Evm, - block: &RecoveredBlock, - receipts: &Vec>, - get_signature: &F, - shared_state: &mut State>, -) -> eyre::Result<()> -where - ChainSpec: EthereumHardforks + EthChainSpec, - F: Fn(&SignedTx) -> (B256, B256, Bytes), -{ - use alloy_consensus::TxReceipt; - - let tracer = &mut *crate::tracer(); - - tracer.on_block_start(firehose_tracer::types::BlockEvent { - block: mapper::to_block_data(block.sealed_block()), - finalized: mapper::to_finalized_ref(ctx.provider().finalized_block_num_hash()), - flash_block: None, - }); - - let evm_env = evm_config - .evm_env(block.header()) - .wrap_err_with(|| format!("Failed to build EVM env for block {}", block.number()))?; - // context_for_block type-checks because FullNodeComponents bounds Node::Evm: - // ConfigureEvm::Primitives> - let exec_ctx = evm_config - .context_for_block(block.sealed_block()) - .wrap_err_with(|| format!("Failed to build EVM context for block {}", block.number()))?; - - // Inspector borrows tracer mutably for the duration of the block execution. - // All tracer lifecycle calls (on_tx_start, on_tx_end, etc.) go through - // executor.evm_mut().inspector_mut().tracer_mut() while the inspector is live. - let inspector = inspector::FirehoseInspector::new(tracer); - // The EVM borrows `shared_state` mutably for the duration of this block. Bundle updates - // from prior blocks in the same ChainCommitted notification are already reflected in its - // cache via earlier `commit_transaction` calls, so nonce / balance reads here see the - // correct pre-block state without re-querying the historical provider. - let evm = evm_config.evm_with_env_and_inspector(&mut *shared_state, evm_env, inspector); - let mut executor = evm_config.create_executor(evm, exec_ctx); - - // AFAIK this is actually unused for tracing, commented out to try, will delete if conclusive - // // Set state hook to log EvmState changes after each transaction and system call - // executor.set_state_hook(Some(Box::new( - // |source: reth_evm::block::StateChangeSource, state: &reth::revm::revm::state::EvmState| { - // inspector::log_evm_state(&format!("state_hook({source:?})"), state); - // }, - // ))); - - // System calls (EIP-4788, EIP-2935, etc.) — handled by apply_pre_execution_changes - executor.evm_mut().inspector_mut().tracer_mut().on_system_call_start(); - executor.apply_pre_execution_changes().wrap_err_with(|| { - format!("Failed to apply pre-execution changes for block {}", block.number()) - })?; - executor.evm_mut().inspector_mut().tracer_mut().on_system_call_end(); - - let mut prev_cumulative_gas: u64 = 0; - let mut log_index: u32 = 0; - - for (tx_index, (recovered_tx, receipt)) in - block.transactions_recovered().zip(receipts.iter()).enumerate() - { - let tx: &SignedTx = &**recovered_tx; - let (r, s, v) = get_signature(tx); - let tx_event = mapper::signed_tx_to_tx_event(tx, recovered_tx.signer(), tx_index, r, s, v); - - // Fresh state reader per transaction for on_tx_start StateReader. - // - // KNOWN LIMITATION: this reader is resolved from the provider at `parent_hash`, not - // from the live `shared_state`. The firehose StateReader observations it produces are - // pre-block, not pre-tx. EVM-level nonce validation is unaffected (that uses the - // shared_state cache, not this reader). - //let state_reader_provider = ctx - // .provider() - // .state_by_block_hash(parent_hash) - // .wrap_err_with(|| { - // format!( - // "Failed to get state reader for block {} tx_index={tx_index} tx_hash={}", - // block.number(), - // recovered_tx.tx_hash() - // ) - // })?; - //let state_reader = Box::new(mapper::StateReaderAdapter(state_reader_provider)); - - executor.evm_mut().inspector_mut().tracer_mut().on_tx_start(tx_event, None); - - let caller_nonce = executor - .evm_mut() - .db_mut() - .basic(recovered_tx.signer())? - .ok_or_else(|| { - eyre::eyre!( - "Failed to get caller account info for block {} tx_index={tx_index} tx_hash={}", - block.number(), - recovered_tx.tx_hash() - ) - })? - .nonce; - info!(target: "firehose", block = block.number(), tx_index, tx_hash = ?recovered_tx.tx_hash(), caller_nonce, "Executing transaction"); - - // execute_transaction_without_commit runs the full EVM execution (including po - // gas refund and miner fee) and returns the final EvmState without committing to DB. - // Inspector hooks (on_call_enter, on_call_exit, on_opcode, etc.) fire during transact(). - let tx_result = - executor.execute_transaction_without_commit(recovered_tx).wrap_err_with(|| { - format!( - "Failed to execute transaction block={} tx_index={tx_index} tx_hash={}", - block.number(), - recovered_tx.tx_hash() - ) - })?; - - // Emit post-execution balance changes (gas refund to sender, miner fee to coinbase). - // revm's post_execution runs reimburse_caller and reward_beneficiary after the last - // inspector hook, so we explicitly compute and emit them using Ethereum gas rules. - { - let result_gas_used = { - use alloy_evm::block::TxResult as _; - tx_result.result().result.tx_gas_used() - }; - let sender = recovered_tx.signer(); - let coinbase = block.header().beneficiary(); - let gas_limit = tx.gas_limit(); - let base_fee = block.header().base_fee_per_gas().unwrap_or(0); - let effective_gas_price: u128 = if tx.is_dynamic_fee() { - std::cmp::min( - tx.max_fee_per_gas(), - base_fee as u128 + tx.max_priority_fee_per_gas().unwrap_or(0), - ) - } else { - tx.gas_price().unwrap_or(0) - }; - - // Committed log count from the ExecutionResult (empty on Revert/Halt). - let committed_log_count = { - use alloy_evm::block::TxResult as _; - tx_result.result().result.logs().len() as u32 - }; - let (db, inspector, _) = executor.evm_mut().components_mut(); - inspector.process_post_tx_balance_changes( - sender, - coinbase, - gas_limit, - result_gas_used, - effective_gas_price, - base_fee, - committed_log_count, - |addr| db.basic(addr).ok().flatten().map(|info| info.balance).unwrap_or(U256::ZERO), - ); - } - - executor.commit_transaction(tx_result); - - let cumulative_gas = receipt.cumulative_gas_used(); - let gas_used = cumulative_gas - prev_cumulative_gas; - let log_count = receipt.logs().len() as u32; - // EIP-4844 receipt fields. `blob_gas_used()` returns `None` for non-blob tx types; - // `blob_gasprice()` returns `None` for pre-Cancun blocks. Matches Geth's receipt shape. - let blob_gas_used = tx.blob_gas_used().unwrap_or(0); - let blob_gas_price = executor.evm().block().blob_gasprice().map(U256::from); - let receipt_data = mapper::to_receipt_data( - receipt, - tx_index as u32, - gas_used, - log_index, - blob_gas_used, - blob_gas_price, - ); - prev_cumulative_gas = cumulative_gas; - log_index += log_count; - - executor.evm_mut().inspector_mut().tracer_mut().on_tx_end(Some(&receipt_data), None); - } - - executor.evm_mut().inspector_mut().tracer_mut().on_system_call_start(); - - // Post-execution changes (block rewards, withdrawals, etc.) - // This consumes the executor, dropping the inspector and releasing the tracer borrow. - // State mutations (pre-execution changes, tx commits, post-execution changes) remain in - // `shared_state`, ready for the next block in this chain notification. - executor.apply_post_execution_changes().wrap_err_with(|| { - format!("Failed to apply post-execution changes for block {}", block.number()) - })?; - - // Tracer borrow released — can call directly again - tracer.on_system_call_end(); - - tracer.on_block_end(None); - - Ok(()) -} - -/// ExEx entry point for Firehose live-block tracing. -/// -/// Loops over `ChainCommitted` notifications, re-executes each block with `trace_block`, and -/// signals `FinishedHeight` after each block so the WAL can be pruned. -pub async fn run_exex(mut ctx: ExExContext) -> eyre::Result<()> -where - Node: FullNodeComponents, - Node::Provider: BlockReader + StateProviderFactory, - ChainSpec: EthereumHardforks + EthChainSpec, - SignedTx: mapper::SignatureFields, -{ - let chain_id = ctx.config.chain.chain().id(); - crate::tracer().on_blockchain_init( - "reth", - env!("CARGO_PKG_VERSION"), - firehose_tracer::config::ChainConfig::new(chain_id), - ); - - emit_genesis_block_if_empty(&ctx)?; - - while let Some(notification) = ctx.notifications.next().await { - let notification = notification?; - - if let Some(committed) = notification.committed_chain() { - info!(chain = ?committed.range(), "Chain committed, tracing {} blocks", committed.len()); - //let first_block = committed.first(); - //let parent_hash = first_block.parent_hash(); - - //let state_provider = - // ctx.provider().state_by_block_hash(parent_hash).wrap_err_with(|| { - // format!("Failed to get state provider for parent block {}", parent_hash) - // })?; - - //let mut _shared_state = State::builder() - // .with_database(StateProviderDatabase::new(state_provider)) - // .with_bundle_update() - // .build(); - - //let _evm_config = ctx.evm_config().clone(); - - for (block, _receipts) in committed.blocks_and_receipts() { - // trace_block( - // &ctx, - // &evm_config, - // block, - // receipts, - // &|tx: &SignedTx| tx.signature_fields(), - // &mut shared_state, - // )?; - - ctx.events.send(ExExEvent::FinishedHeight(block.num_hash()))?; - } - } - } - - Ok(()) -} - -/// Emits the genesis block (block 0) to the Firehose stream if the chain is empty (head still -/// at genesis). -/// -/// Reth never executes the genesis block — it is written straight to the database during init — -/// so no execution hook ever fires for it. Without this, a Firehose stream started from scratch -/// begins at block 1 and downstream consumers waiting for block 0 stall forever. Mirrors geth's -/// `OnGenesisBlock` hook: emitted with the full genesis alloc, only when starting on an empty -/// chain. -pub fn emit_genesis_block_if_empty(ctx: &ExExContext) -> eyre::Result<()> -where - Node: FullNodeComponents, - Node::Provider: BlockReader, - ChainSpec: EthereumHardforks + EthChainSpec, -{ - emit_genesis_block_on_empty_chain(ctx.provider(), ctx.config.chain.genesis()) -} - -/// Provider-based variant of [`emit_genesis_block_if_empty`] for embedders (op-reth, base) that -/// wire Firehose through their own engine validator instead of this crate's ExEx. Call it once -/// at node startup, after `on_blockchain_init` and before any block is traced. -pub fn emit_genesis_block_on_empty_chain

( - provider: &P, - genesis: &alloy_genesis::Genesis, -) -> eyre::Result<()> -where - P: BlockReader, - ::Header: BlockHeader + alloy_primitives::Sealable, - <::Body as reth_primitives_traits::BlockBody>::OmmerHeader: - BlockHeader + alloy_primitives::Sealable, -{ - use reth_primitives_traits::Block as _; - - // OP-stack chains may place their genesis at a non-zero height (e.g. OP mainnet's - // post-bedrock genesis); `genesis.number` carries it, defaulting to 0. - let genesis_number = genesis.number.unwrap_or(0); - if provider.best_block_number()? != genesis_number { - return Ok(()); - } - - let genesis_block = provider - .block(genesis_number.into())? - .ok_or_else(|| eyre::eyre!("genesis block {genesis_number} not found in database"))? - .seal_slow(); - - crate::tracer().on_genesis_block( - firehose_tracer::types::BlockEvent { - block: mapper::to_block_data(&genesis_block), - // Genesis is finalized by definition. - finalized: Some(firehose_tracer::types::FinalizedBlockRef { - number: genesis_number, - hash: Some(genesis_block.hash()), - }), - flash_block: None, - }, - mapper::to_genesis_alloc(genesis), - ); - - Ok(()) -}