Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2,996 changes: 1,809 additions & 1,187 deletions Cargo.lock

Large diffs are not rendered by default.

99 changes: 55 additions & 44 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -70,62 +70,73 @@ codegen-units = 1
incremental = false

[workspace.dependencies]
reth = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-db = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-db-common = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-node-api = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-node-core = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-provider = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646", features = [
# default-features minus "rocksdb": we only read reth datadirs via MDBX + static files.
# ("reth-revm/portable" from reth's default set is re-enabled on the reth-revm dep below.)
reth = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b", default-features = false, features = [
"jemalloc",
"otlp",
"otlp-logs",
"js-tracer",
"keccak-cache-global",
"asm-keccak",
"min-debug-logs",
] }
reth-cli-util = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-db = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-db-common = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-errors = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-payload-builder = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-node-api = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-node-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-trie = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-trie-parallel = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-node-core = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-ethereum-primitives = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-provider = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b", features = [
"test-utils",
] }
reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-evm = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-revm = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-trie-db = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }
reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "27a8c0f5a6dfb27dea84c5751776ecabdd069646" }

# compatible with reth 27a8c0f5a6dfb27dea84c5751776ecabdd069646 dependencies
revm = { version = "31.0.2", features = [
reth-chainspec = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-evm = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-evm-ethereum = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-revm = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b", features = ["portable"] }
reth-execution-errors = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-trie-db = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-transaction-pool = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }
reth-tasks = { git = "https://github.com/paradigmxyz/reth", rev = "d6324d63e27ef6b7c49cdc9b1977c1b808234c7b" }

# compatible with reth v1.11.3 (d6324d63e27ef6b7c49cdc9b1977c1b808234c7b) dependencies
revm = { version = "34.0.0", features = [
"std",
"secp256k1",
"optional_balance_check",
], default-features = false }
revm-inspectors = { version = "0.32.0", default-features = false }
revm-inspectors = { version = "0.34.2", default-features = false }

ethereum_ssz_derive = "0.9.0"
ethereum_ssz = "0.9.0"

alloy-primitives = { version = "1.4.1", default-features = false, features = [
alloy-primitives = { version = "1.5.6", default-features = false, features = [
"getrandom",
] }
alloy-rlp = "0.3.10"
alloy-rlp = "0.3.13"
alloy-chains = "0.2.5"
alloy-trie = { version = "0.8.1", default-features = false }
alloy-evm = { version = "0.23.3", default-features = false }
alloy-provider = { version = "1.0.41", features = ["ipc", "pubsub", "ws"] }
alloy-eips = { version = "1.0.41" }
alloy-rpc-types = { version = "1.0.41" }
alloy-json-rpc = { version = "1.0.41" }
alloy-network = { version = "1.0.41" }
alloy-network-primitives = { version = "1.0.41" }
alloy-node-bindings = { version = "1.0.41" }
alloy-consensus = { version = "1.0.41", features = ["kzg"] }
alloy-rpc-types-beacon = { version = "1.0.41", features = ["ssz"] }
alloy-rpc-types-engine = { version = "1.0.41", features = ["ssz"] }
alloy-rpc-types-eth = { version = "1.0.41" }
alloy-signer = { version = "1.0.41" }
alloy-signer-local = { version = "1.0.41" }
alloy-trie = { version = "0.9.4", default-features = false }
alloy-evm = { version = "0.27.2", default-features = false }
alloy-provider = { version = "1.6.3", features = ["ipc", "pubsub", "ws"] }
alloy-eips = { version = "1.6.3" }
alloy-rpc-types = { version = "1.6.3" }
alloy-json-rpc = { version = "1.6.3" }
alloy-network = { version = "1.6.3" }
alloy-network-primitives = { version = "1.6.3" }
alloy-node-bindings = { version = "1.6.3" }
alloy-consensus = { version = "1.6.3", features = ["kzg"] }
alloy-rpc-types-beacon = { version = "1.6.3", features = ["ssz"] }
alloy-rpc-types-engine = { version = "1.6.3", features = ["ssz"] }
alloy-rpc-types-eth = { version = "1.6.3" }
alloy-signer = { version = "1.6.3" }
alloy-signer-local = { version = "1.6.3" }

# Version required by ethereum-consensus beacon-api-client
mev-share-sse = { git = "https://github.com/paradigmxyz/mev-share-rs", rev = "9eb2b0138ab3202b9eb3af4b19c7b3bf40b0faa8", default-features = false }
Expand All @@ -141,7 +152,7 @@ parking_lot = { version = "0.12.3" }
tokio = { version = "1.40.0", default-features = false }
futures = "0.3"
futures-util = "0.3"
reqwest = { version = "0.12.8" }
reqwest = { version = "0.12.8", features = ["json"] }
serde = { version = "1.0.210" }
serde_json = { version = "1.0" }
serde_with = { version = "3.8.1" }
Expand Down
4 changes: 3 additions & 1 deletion crates/eth-sparse-mpt/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,7 @@ rustc-hash = "2.0.0"
rayon = "1.10.0"
smallvec = "1.13.2"
alloy-trie.workspace = true
nybbles = { version = "0.3.3", features = ["serde"] }
nybbles = { version = "0.4", features = ["serde"] }

tracing.workspace = true

Expand All @@ -32,6 +32,7 @@ reth-node-api = { workspace = true, optional = true }
reth-node-ethereum = { workspace = true, optional = true }
reth-trie.workspace = true
reth-trie-parallel = { workspace = true, optional = true }
reth-tasks = { workspace = true, optional = true }
reth-trie-db.workspace = true
reth-provider.workspace = true
reth-revm = { workspace = true, optional = true }
Expand Down Expand Up @@ -64,6 +65,7 @@ dev-tools = [
"dep:reth-node-api",
"dep:reth-node-ethereum",
"dep:reth-trie-parallel",
"dep:reth-tasks",
"dep:reth-revm",
]

Expand Down
8 changes: 4 additions & 4 deletions crates/eth-sparse-mpt/benches/trie_do_bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -92,14 +92,14 @@ fn insert_proofs(c: &mut Criterion) {
let mut proofs: HashMap<Nibbles, Vec<(Nibbles, Bytes)>> = Default::default();
for key in &byte_keys {
let key = Nibbles::unpack(key);
let current_key_proofs = proofs.entry(key.clone()).or_default();
let current_key_proofs = proofs.entry(key).or_default();
for (path, node) in &nodes {
if key.starts_with(path) {
current_key_proofs.push((path.clone(), node.clone().into()));
current_key_proofs.push((*path, node.clone().into()));
}
}
current_key_proofs.sort_by_key(|(p, _)| p.clone());
current_key_proofs.dedup_by_key(|(p, _)| p.clone());
current_key_proofs.sort_by_key(|(p, _)| *p);
current_key_proofs.dedup_by_key(|(p, _)| *p);
}
for (path, proof) in proofs {
proof_store.add_proof(path, proof).unwrap();
Expand Down
4 changes: 2 additions & 2 deletions crates/eth-sparse-mpt/benches/trie_insert_bench.rs
Original file line number Diff line number Diff line change
Expand Up @@ -120,8 +120,8 @@ fn gather_nodes(c: &mut Criterion) {
.into_iter()
.flat_map(|mp| mp.account_subtree.into_iter().collect::<Vec<_>>())
.collect();
account_proof.sort_by_key(|(p, _)| p.clone());
account_proof.dedup_by_key(|(p, _)| p.clone());
account_proof.sort_by_key(|(p, _)| *p);
account_proof.dedup_by_key(|(p, _)| *p);
account_proof
};

Expand Down
54 changes: 40 additions & 14 deletions crates/eth-sparse-mpt/src/bin/correctness-harness.rs
Original file line number Diff line number Diff line change
Expand Up @@ -14,12 +14,16 @@ use reth_evm_ethereum::EthEvmConfig;
use reth_node_api::NodeTypesWithDBAdapter;
use reth_node_ethereum::EthereumNode;
use reth_provider::{
providers::{ConsistentDbView, OverlayStateProviderFactory, StaticFileProvider},
providers::{
ConsistentDbView, OverlayStateProviderFactory, RocksDBProvider, StaticFileProvider,
StaticFileProviderBuilder,
},
BlockHashReader, BlockNumReader, BlockReader, ChainSpecProvider, HeaderProvider,
ProviderFactory, TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_trie::TrieInput;
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::root::ParallelStateRoot;
use revm::database::BundleState;
use std::{
Expand Down Expand Up @@ -347,13 +351,13 @@ fn calculate_reth_root(
parent_hash: B256,
outcome: &BundleState,
) -> Result<B256> {
let overlay = OverlayStateProviderFactory::new(factory.clone());
let overlay = OverlayStateProviderFactory::new(factory.clone(), ChangesetCache::new());
let hasher = factory
.history_by_block_hash(parent_hash)
.with_context(|| format!("failed to open state provider at parent hash {parent_hash:?}"))?;
let hashed_post_state = hasher.hashed_post_state(outcome);
let trie_input = TrieInput::from_state(hashed_post_state);
ParallelStateRoot::new(overlay, trie_input.prefix_sets.freeze())
ParallelStateRoot::new(overlay, trie_input.prefix_sets.freeze(), task_runtime())
.incremental_root()
.with_context(|| "parallel state root failed")
}
Expand Down Expand Up @@ -403,24 +407,46 @@ fn open_provider_factory(
.with_context(|| format!("failed to open reth db at {}", db_path.display()))?,
);

let static_files =
StaticFileProvider::read_only(&static_files_path, false).with_context(|| {
format!(
"failed to open static files at {}",
static_files_path.display()
)
})?;
let static_files = if let Some(blocks_per_file) = static_file_blocks_per_file {
static_files.with_custom_blocks_per_file(blocks_per_file)
StaticFileProviderBuilder::read_only(&static_files_path)
.with_blocks_per_file(blocks_per_file)
.build()
} else {
static_files
};
StaticFileProvider::read_only(&static_files_path, false)
}
.with_context(|| {
format!(
"failed to open static files at {}",
static_files_path.display()
)
})?;

let rocksdb_provider = RocksDBProvider::builder(datadir.join("rocksdb"))
.with_default_tables()
.with_read_only(true)
.build()
.context("failed to open rocksdb provider")?;
Ok(ProviderFactory::<NodeTypes>::new(
db,
chain_spec,
static_files,
))
rocksdb_provider,
task_runtime(),
)?)
}

/// Process-wide reth task runtime for parallel provider/trie I/O.
fn task_runtime() -> reth_tasks::Runtime {
use reth_tasks::{Runtime, RuntimeBuilder, RuntimeConfig};

static RUNTIME: std::sync::OnceLock<Runtime> = std::sync::OnceLock::new();
RUNTIME
.get_or_init(|| {
RuntimeBuilder::new(RuntimeConfig::default())
.build()
.expect("failed to build reth task runtime")
})
.clone()
}

fn parse_chain_spec(chain: &str) -> Result<Arc<ChainSpec>> {
Expand Down
27 changes: 6 additions & 21 deletions crates/eth-sparse-mpt/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -29,25 +29,23 @@ pub fn rlp_pointer(rlp_encode: Bytes) -> Bytes {
}

pub fn concat_path(p1: &Nibbles, p2: &[u8]) -> Nibbles {
let mut result = Nibbles::with_capacity(p1.len() + p2.len());
result.extend_from_slice_unchecked(p1);
let mut result = *p1;
result.extend_from_slice_unchecked(p2);
result
}

pub fn strip_first_nibble_mut(p: &mut Nibbles) -> u8 {
let nibble = p[0];
let vec = p.as_mut_vec_unchecked();
vec.remove(0);
let nibble = p.get_unchecked(0);
*p = p.slice_unchecked(1, p.len());
nibble
}

#[inline]
pub fn extract_prefix_and_suffix(p1: &Nibbles, p2: &Nibbles) -> (Nibbles, Nibbles, Nibbles) {
let prefix_len = p1.common_prefix_length(p2);
let prefix = Nibbles::from_nibbles_unchecked(&p1[..prefix_len]);
let suffix1 = Nibbles::from_nibbles_unchecked(&p1[prefix_len..]);
let suffix2 = Nibbles::from_nibbles_unchecked(&p2[prefix_len..]);
let prefix = p1.slice_unchecked(0, prefix_len);
let suffix1 = p1.slice_unchecked(prefix_len, p1.len());
let suffix2 = p2.slice_unchecked(prefix_len, p2.len());

(prefix, suffix1, suffix2)
}
Expand Down Expand Up @@ -136,16 +134,3 @@ fn mismatch_chunks<const N: usize>(xs: &[u8], ys: &[u8]) -> usize {
.take_while(|(x, y)| x == y)
.count()
}

// rbuilder uses nybbles v3.3.0 and reth_trie uses nybbles v4.3.0. This is a temporary fix to convert between the two.
// We can remove the below methods once rbuilder has been upgraded to nybbles v4.3.0.
// nybbles v4.3.0 has a breaking change (byte array with 1 byte per nybble vs U256 packed data + length) in the API which breaks a lot of parts of the eth sparse trie code.
#[inline]
pub fn convert_reth_nybbles_to_nibbles(n: reth_trie::Nibbles) -> Nibbles {
Nibbles::from_nibbles(n.to_vec())
}

#[inline]
pub fn convert_nibbles_to_reth_nybbles(n: Nibbles) -> reth_trie::Nibbles {
reth_trie::Nibbles::from_nibbles(n.as_slice())
}
Original file line number Diff line number Diff line change
Expand Up @@ -146,11 +146,11 @@ impl RethSparseTrieShareCacheInternal {
multiproof: MultiProof,
) -> Result<(), AddNodeError> {
let mut nodes: Vec<_> = multiproof.account_subtree.into_iter().collect();
nodes.sort_by_key(|(p, _)| p.clone());
nodes.sort_by_key(|(p, _)| *p);
self.account_trie.add_nodes(&nodes)?;
for (account, storge_proofs) in multiproof.storages {
let mut nodes: Vec<_> = storge_proofs.subtree.into_iter().collect();
nodes.sort_by_key(|(p, _)| p.clone());
nodes.sort_by_key(|(p, _)| *p);
let account = Bytes::copy_from_slice(account.as_slice());
let storage_trie = self.storage_tries.entry(account).or_default();
storage_trie.add_nodes(&nodes)?;
Expand Down
22 changes: 12 additions & 10 deletions crates/eth-sparse-mpt/src/v1/reth_sparse_trie/trie_fetcher/mod.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use std::time::Instant;

use crate::utils::{convert_reth_nybbles_to_nibbles, hash_map_with_capacity, HashMap, HashSet};
use crate::utils::{hash_map_with_capacity, HashMap, HashSet};
use alloy_primitives::map::HashSet as AlloyHashSet;
use tracing::trace;

Expand Down Expand Up @@ -106,7 +106,9 @@ where
}

fn pad_path(mut path: Nibbles) -> B256 {
path.as_mut_vec_unchecked().resize(64, 0);
while path.len() < 64 {
path.push_unchecked(0);
}
let mut res = B256::default();
path.pack_to(res.as_mut_slice());
res
Expand Down Expand Up @@ -164,16 +166,16 @@ fn merge_results(
let mut result = MultiProof::default();
for mut proof in multiproofs {
result.account_subtree.append(&mut proof.account_subtree);
result.account_subtree.sort_by_key(|s| s.0.clone());
result.account_subtree.dedup_by_key(|s| s.0.clone());
result.account_subtree.sort_by_key(|s| s.0);
result.account_subtree.dedup_by_key(|s| s.0);

for (account, mut storage_proof) in proof.storages {
let result_storage_proof = result.storages.entry(account).or_default();
result_storage_proof
.subtree
.append(&mut storage_proof.subtree);
result_storage_proof.subtree.sort_by_key(|s| s.0.clone());
result_storage_proof.subtree.dedup_by_key(|s| s.0.clone());
result_storage_proof.subtree.sort_by_key(|s| s.0);
result_storage_proof.subtree.dedup_by_key(|s| s.0);
}
}

Expand All @@ -192,9 +194,9 @@ fn convert_reth_multiproof(
) -> MultiProof {
let mut account_subtree = Vec::with_capacity(reth_proof.account_subtree.len());
for (k, v) in reth_proof.account_subtree.into_inner() {
account_subtree.push((convert_reth_nybbles_to_nibbles(k), v));
account_subtree.push((k, v));
}
account_subtree.sort_by_key(|a| a.0.clone());
account_subtree.sort_by_key(|a| a.0);
let mut storages = hash_map_with_capacity(reth_proof.storages.len());
for (k, reth_storage_proof) in reth_proof.storages {
if !all_requested_accounts.contains(&k) {
Expand All @@ -206,9 +208,9 @@ fn convert_reth_multiproof(
let mut subtree = Vec::with_capacity(reth_storage_proof.subtree.len());

for (k, v) in reth_storage_proof.subtree.into_inner() {
subtree.push((convert_reth_nybbles_to_nibbles(k), v));
subtree.push((k, v));
}
subtree.sort_by_key(|a| a.0.clone());
subtree.sort_by_key(|a| a.0);
let v = StorageMultiProof { subtree };
storages.insert(k, v);
}
Expand Down
Loading
Loading