diff --git a/Cargo.lock b/Cargo.lock index c15c1d4..d37f07f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1321,6 +1321,7 @@ dependencies = [ "pallet-contracts", "pallet-contracts-primitives", "parity-scale-codec", + "scale-decode 0.9.0", "serde", "sp-core", "sp-version", @@ -2772,7 +2773,7 @@ dependencies = [ "derive_more", "ink_prelude", "parity-scale-codec", - "scale-decode", + "scale-decode 0.5.0", "scale-encode", "scale-info", "xxhash-rust", @@ -4461,6 +4462,16 @@ dependencies = [ "scale-info", ] +[[package]] +name = "scale-bits" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "036575c29af9b6e4866ffb7fa055dbf623fe7a9cc159b33786de6013a6969d89" +dependencies = [ + "parity-scale-codec", + "scale-info", +] + [[package]] name = "scale-decode" version = "0.5.0" @@ -4468,12 +4479,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c7e5527e4b3bf079d4c0b2f253418598c380722ba37ef20fac9088081407f2b6" dependencies = [ "parity-scale-codec", - "scale-bits", - "scale-decode-derive", + "scale-bits 0.3.0", + "scale-decode-derive 0.5.0", "scale-info", "thiserror", ] +[[package]] +name = "scale-decode" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7789f5728e4e954aaa20cadcc370b99096fb8645fca3c9333ace44bb18f30095" +dependencies = [ + "derive_more", + "parity-scale-codec", + "primitive-types", + "scale-bits 0.4.0", + "scale-decode-derive 0.9.0", + "scale-info", + "smallvec", +] + [[package]] name = "scale-decode-derive" version = "0.5.0" @@ -4487,6 +4513,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "scale-decode-derive" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27873eb6005868f8cc72dcfe109fae664cf51223d35387bc2f28be4c28d94c47" +dependencies = [ + "darling 0.14.4", + "proc-macro-crate 1.3.1", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "scale-encode" version = "0.1.2" diff --git a/Cargo.toml b/Cargo.toml index a0afa4e..5e338e9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,5 +1,6 @@ [workspace] members = ["crates/*"] +resolver = "2" [workspace.package] version = "0.1.0" diff --git a/crates/common/Cargo.toml b/crates/common/Cargo.toml index e5d2f4b..713baab 100644 --- a/crates/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -21,6 +21,7 @@ frame-metadata = { version = "15.1", default-features = false, features = ["v14" parity-scale-codec = { version = "3.6.3", optional = true } pallet-contracts = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false, optional = true } pallet-contracts-primitives = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false, optional = true } +scale-decode = { version = "0.9.0", optional = true } sp-core = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false, optional = true } sp-version = { git = "https://github.com/paritytech/substrate", branch = "polkadot-v0.9.43", default-features = false, optional = true } substrate-api-client = { git = "https://github.com/scs/substrate-api-client", branch = "polkadot-v0.9.43", default-features = false, features = ["jsonrpsee-client", "contracts-xt"], optional = true } @@ -35,6 +36,7 @@ rpc = [ "parity-scale-codec", "pallet-contracts", "pallet-contracts-primitives", + "scale-decode", "sp-core", "sp-version", "substrate-api-client" diff --git a/crates/common/src/rpc.rs b/crates/common/src/rpc.rs index 23207e6..5ccd67c 100644 --- a/crates/common/src/rpc.rs +++ b/crates/common/src/rpc.rs @@ -14,15 +14,15 @@ use std::{convert::identity, num::NonZeroUsize}; -use frame_metadata::RuntimeMetadataPrefixed; +use frame_metadata::{RuntimeMetadataPrefixed, StorageEntryType}; use futures_util::{ stream::{self, try_unfold}, Stream, StreamExt, TryStreamExt, }; use lru::LruCache; -use pallet_contracts::Determinism; use pallet_contracts_primitives::ContractExecResult; -use parity_scale_codec::{Compact, Decode, Encode}; +use parity_scale_codec::{Decode, Encode}; +use scale_decode::DecodeAsType; use sp_core::crypto::AccountId32; use sp_version::RuntimeVersion; use substrate_api_client::{ @@ -43,27 +43,17 @@ pub use substrate_api_client; pub const PAGE_SIZE: u32 = 10; /// WASM blob information received from an RPC node. -#[derive(Decode)] +#[derive(DecodeAsType)] struct PrefabWasmModule { - _instruction_weights_version: Compact, - _initial: Compact, - _maximum: Compact, /// WASM bytecode value. code: Vec, - _determinism: Determinism, } /// Deployed contract information from an RPC node. -#[derive(Decode)] +#[derive(DecodeAsType)] pub struct ContractInfo { - _trie_id: Vec, /// Code hash associated with the current contract. pub code_hash: H256, - _storage_bytes: u32, - _storage_items: u32, - _storage_byte_deposit: u128, - _storage_item_deposit: u128, - _storage_base_deposit: u128, } /// Get a [`Block`] information for the provided block hash. @@ -82,15 +72,20 @@ pub async fn block( /// /// This method returns an asynchronous [`Stream`] of [`StorageKey`] (which can be decoded to receive the code hash value) /// and WASM blob bytes. -pub async fn pristine_code_root( - api: &Api, +pub async fn pristine_code_root<'a, C: Request>( + api: &'a Api, at: H256, -) -> Result)>, Error>> + '_, Error> { - let prefix = api - .get_storage_map_key_prefix("Contracts", "CodeStorage") - .await?; - - Ok(paged_key_values::<_, PrefabWasmModule, _, _>(prefix, api, at, |module| module.code).await) + metadata: &'a Metadata, +) -> Result)>, Error>> + 'a, Error> { + paged_key_values::<_, PrefabWasmModule, _, _>( + api, + "Contracts", + "PristineCode", + at, + |module| module.code, + metadata, + ) + .await } /// Get WASM blob for the provided code hash at the provided block hash. @@ -100,25 +95,30 @@ pub async fn pristine_code( api: &Api, at: H256, code_hash: H256, + metadata: &Metadata, ) -> Result>, Error> { - api.get_storage_map::<_, PrefabWasmModule>("Contracts", "CodeStorage", code_hash, Some(at)) - .await - .map(|val| val.map(|module| module.code)) + get_ty_storage_by_key::<_, _, PrefabWasmModule>( + api, + "Contracts", + "PristineCode", + code_hash, + at, + metadata, + ) + .await + .map(|val| val.map(|module| module.code)) } /// Get information on all available contracts at the provided block hash. /// /// This method returns an asynchronous [`Stream`] of [`StorageKey`] (which can be decoded to receive the contract address value) /// and associated contract information. -pub async fn contract_info_of_root( - api: &Api, +pub async fn contract_info_of_root<'a, C: Request + Send + Sync>( + api: &'a Api, at: H256, -) -> Result, Error>> + '_, Error> { - let prefix = api - .get_storage_map_key_prefix("Contracts", "ContractInfoOf") - .await?; - - Ok(paged_key_values(prefix, api, at, identity).await) + metadata: &'a Metadata, +) -> Result, Error>> + 'a, Error> { + paged_key_values(api, "Contracts", "ContractInfoOf", at, identity, metadata).await } /// Get information about the specific contract at the provided block hash. @@ -128,9 +128,9 @@ pub async fn contract_info_of( api: &Api, at: H256, account_id: &AccountId32, + metadata: &Metadata, ) -> Result, Error> { - api.get_storage_map("Contracts", "ContractInfoOf", account_id, Some(at)) - .await + get_ty_storage_by_key(api, "Contracts", "ContractInfoOf", account_id, at, metadata).await } /// Get UNIX timestamp in milliseconds for the provided block hash. @@ -207,11 +207,11 @@ impl MetadataCache { /// /// This method requests node runtime version corresponding to the provided block, /// and either fetches it from node or retrieves from cache. - pub async fn metadata( - &mut self, + pub async fn metadata<'a, C: Request>( + &'a mut self, api: &Api, at: H256, - ) -> Result { + ) -> Result<&'a Metadata, Error> { let RuntimeVersion { authoring_version, spec_version, @@ -222,12 +222,10 @@ impl MetadataCache { .request("state_getRuntimeVersion", rpc_params![at]) .await?; - if let Some(metadata) = self + if !self .cache - .get(&(authoring_version, spec_version, impl_version)) + .contains(&(authoring_version, spec_version, impl_version)) { - Ok(metadata.clone()) - } else { let metadata_bytes: Bytes = api .client() .request("state_getMetadata", rpc_params![Some(at)]) @@ -241,9 +239,14 @@ impl MetadataCache { (authoring_version, spec_version, impl_version), metadata.clone(), ); - - Ok(metadata) } + + let metadata = self + .cache + .get(&(authoring_version, spec_version, impl_version)) + .unwrap(); + + Ok(metadata) } } @@ -258,11 +261,11 @@ impl Default for MetadataCache { /// Fetch events associated with the provided block hash. /// /// Since events layout may differ between different runtime upgrades, -/// this method accepts [`MetadataCache`] to correctly query node for the corresponding metadata. +/// this method accepts [`Metadata`] to correctly query node. pub async fn events( api: &Api, at: H256, - metadata_cache: &mut MetadataCache, + metadata: Metadata, ) -> Result, Error> { let key = storage_key("System", "Events"); let event_bytes = api @@ -270,11 +273,7 @@ pub async fn events( .await? .ok_or(Error::BlockNotFound)?; - Ok(Events::new( - metadata_cache.metadata(api, at).await?, - Default::default(), - event_bytes, - )) + Ok(Events::new(metadata, Default::default(), event_bytes)) } /// Contract instantiation event. @@ -335,16 +334,36 @@ impl StaticEvent for Terminated { const EVENT: &'static str = "Terminated"; } -// Get storage keys and values with the provided prefix, mapping values in process. -async fn paged_key_values T + 'static>( - prefix: StorageKey, +async fn get_ty_storage_by_key( api: &Api, + pallet: &'static str, + storage_item: &'static str, + map_key: K, + at: H256, + metadata: &Metadata, +) -> Result, Error> { + let storage_key = metadata.storage_map_key(pallet, storage_item, map_key)?; + + api.get_opaque_storage_by_key(storage_key, Some(at)) + .await? + .map(|input| resolve_ty(metadata, pallet, storage_item, &mut &*input)) + .transpose() +} + +// Get storage keys and values with the provided prefix, mapping values in process. +async fn paged_key_values<'a, C: Request, V: DecodeAsType, T, F: FnMut(V) -> T + 'static>( + api: &'a Api, + pallet: &'static str, + storage_item: &'static str, at: H256, map: F, -) -> impl Stream, Error>> + '_ { - try_unfold( - (None, prefix, map), - move |(start_key, prefix, mut map)| async move { + metadata: &'a Metadata, +) -> Result, Error>> + 'a, Error> { + let prefix = api.get_storage_map_key_prefix(pallet, storage_item).await?; + + Ok(try_unfold( + (None, prefix, map, metadata), + move |(start_key, prefix, mut map, metadata)| async move { let storage_keys = api .get_storage_keys_paged(Some(prefix.clone()), PAGE_SIZE, start_key, Some(at)) .await?; @@ -358,8 +377,10 @@ async fn paged_key_values T + 'static>( let values = stream::iter(storage_keys) .then(move |storage_key| async move { let value = api - .get_storage_by_key(storage_key.clone(), Some(at)) + .get_opaque_storage_by_key(storage_key.clone(), Some(at)) .await? + .map(|input| resolve_ty(metadata, pallet, storage_item, &mut &*input)) + .transpose()? .expect("unable to find value corresponding to the provided storage key"); Result::<_, Error>::Ok((storage_key, value)) @@ -368,7 +389,24 @@ async fn paged_key_values T + 'static>( .try_collect() .await?; - Result::<_, Error>::Ok(Some((values, (start_key, prefix, map)))) + Result::<_, Error>::Ok(Some((values, (start_key, prefix, map, metadata)))) }, - ) + )) +} + +fn resolve_ty( + metadata: &Metadata, + pallet_name: &'static str, + storage_key: &'static str, + input: &mut &[u8], +) -> Result { + let type_id = match metadata.pallet(pallet_name)?.storage(storage_key)?.ty { + StorageEntryType::Plain(ty) => ty.id, + StorageEntryType::Map { value, .. } => value.id, + }; + + let ty = T::decode_as_type(input, type_id, metadata.types()) + .expect("unable to parse DecodeAsType type"); + + Ok(ty) } diff --git a/crates/event_client/src/cli/initialize.rs b/crates/event_client/src/cli/initialize.rs index 3f6d9b8..4a93a20 100644 --- a/crates/event_client/src/cli/initialize.rs +++ b/crates/event_client/src/cli/initialize.rs @@ -4,6 +4,7 @@ use common::rpc::{ self, sp_core::crypto::AccountId32, substrate_api_client::{self, ac_primitives::Block, rpc::JsonrpseeClient, Api}, + MetadataCache, }; use db::{ code, contract, node, sea_query::OnConflict, ActiveValue, DatabaseConnection, DbErr, @@ -50,12 +51,16 @@ pub async fn initialize( let client = JsonrpseeClient::new(&url).map_err(substrate_api_client::Error::RpcClient)?; let api = Api::new(client).await?; + let mut metadata_cache = MetadataCache::new(); + let latest_block = rpc::block(&api, None) .await? .expect("at least one block is expected"); let block_hash = latest_block.hash(); + let metadata = metadata_cache.metadata(&api, block_hash).await?; + let payment_address = payment_address .as_deref() .map(AccountId32::from_str) @@ -91,7 +96,7 @@ pub async fn initialize( .await .into_raw_result()?; - let mut wasm_blobs = pin!(rpc::pristine_code_root(&api, block_hash).await?); + let mut wasm_blobs = pin!(rpc::pristine_code_root(&api, block_hash, metadata).await?); while let Some(chunk) = wasm_blobs.try_next().await? { database @@ -118,7 +123,7 @@ pub async fn initialize( .into_raw_result()?; } - let mut contracts = pin!(rpc::contract_info_of_root(&api, block_hash).await?); + let mut contracts = pin!(rpc::contract_info_of_root(&api, block_hash, metadata).await?); while let Some(chunk) = contracts.try_next().await? { database diff --git a/crates/event_client/src/cli/traverse.rs b/crates/event_client/src/cli/traverse.rs index 34c2730..9ebaa6a 100644 --- a/crates/event_client/src/cli/traverse.rs +++ b/crates/event_client/src/cli/traverse.rs @@ -104,7 +104,9 @@ async fn parse_block( block_hash: H256, metadata_cache: &mut MetadataCache, ) -> Result { - let events = rpc::events(api, block_hash, metadata_cache).await?; + let metadata = metadata_cache.metadata(api, block_hash).await?; + + let events = rpc::events(api, block_hash, metadata.clone()).await?; let instantiations = events.find().try_collect()?; diff --git a/crates/event_client/src/cli/watch.rs b/crates/event_client/src/cli/watch.rs index 0ef6af8..829d947 100644 --- a/crates/event_client/src/cli/watch.rs +++ b/crates/event_client/src/cli/watch.rs @@ -5,6 +5,7 @@ use common::rpc::{ sp_core::ByteArray, substrate_api_client::{ self, + ac_node_api::Metadata, ac_primitives::{Block, Config, Header, PolkadotConfig}, rpc::{HandleSubscription, JsonrpseeClient, Request}, Api, GetChainInfo, SubscribeChain, @@ -83,7 +84,8 @@ pub async fn watch(database: DatabaseConnection, name: String) -> Result<(), Wat while let Some(block) = stream.try_next().await? { debug!(block_number = %block.header().number(), "found a block to catch-up to"); - node = process_block(node, &database, &api, block.header(), &mut metadata_cache).await?; + let metadata = metadata_cache.metadata(&api, block.hash()).await?; + node = process_block(node, &database, &api, block.header(), metadata).await?; } // Proceed with the subscription, since an attempt to traverse missed blocks was already made. @@ -99,7 +101,8 @@ pub async fn watch(database: DatabaseConnection, name: String) -> Result<(), Wat .map_err(substrate_api_client::Error::RpcClient)? { debug!(block_number = %header.number(), "found new block"); - node = process_block(node, &database, &api, &header, &mut metadata_cache).await?; + let metadata = metadata_cache.metadata(&api, header.hash()).await?; + node = process_block(node, &database, &api, &header, metadata).await?; } Ok(()) @@ -115,7 +118,7 @@ async fn process_block( database: &DatabaseConnection, api: &Api, block_header: &::Header, - metadata_cache: &mut MetadataCache, + metadata: &Metadata, ) -> Result { let mut active_node: node::ActiveModel = node.clone().into(); @@ -128,12 +131,12 @@ async fn process_block( .expect("invalid timestamp was provided"); let block_timestamp = PrimitiveDateTime::new(offset_timestamp.date(), offset_timestamp.time()); - let events = rpc::events(api, block_hash, metadata_cache).await?; + let events = rpc::events(api, block_hash, metadata.clone()).await?; let code_uploads = stream::iter(events.find::()) .err_into() .and_then(|CodeStored { code_hash }| async move { - rpc::pristine_code(api, block_hash, code_hash) + rpc::pristine_code(api, block_hash, code_hash, metadata) .await .map(|code| (code_hash.0, code)) }) @@ -148,7 +151,7 @@ async fn process_block( let instantiations = stream::iter(events.find::()) .err_into() .and_then(|Instantiated { deployer, contract }| async move { - rpc::contract_info_of(api, block_hash, &contract) + rpc::contract_info_of(api, block_hash, &contract, metadata) .await .map(|info| (contract, deployer, info)) }) diff --git a/crates/event_client/src/utils.rs b/crates/event_client/src/utils.rs index 245caa4..f9f2b60 100644 --- a/crates/event_client/src/utils.rs +++ b/crates/event_client/src/utils.rs @@ -39,7 +39,7 @@ pub(crate) fn block_mapping_stream<'a, I: IntoIterator + 'a, C: Requ range: I, api: &'a Api, ) -> impl Stream> + 'a { - stream::iter(range.into_iter()) + stream::iter(range) .map(Ok) .try_filter_map(move |block_number| async move { Ok(api diff --git a/crates/patron/src/archiver.rs b/crates/patron/src/archiver.rs index effdbe2..ace5477 100644 --- a/crates/patron/src/archiver.rs +++ b/crates/patron/src/archiver.rs @@ -44,7 +44,10 @@ pub(crate) fn build_zip_archive( while let Some(entry) = entries.next().transpose()? { let Some(path) = entry.path().strip_prefix(¤t_dir)?.to_str() else { - progress.println(format!("File {} contains non-unicode symbols in path", entry.path().display())); + progress.println(format!( + "File {} contains non-unicode symbols in path", + entry.path().display() + )); continue; }; diff --git a/flake.lock b/flake.lock index e731ada..b29bbf0 100644 --- a/flake.lock +++ b/flake.lock @@ -14,11 +14,11 @@ ] }, "locked": { - "lastModified": 1688772518, - "narHash": "sha256-ol7gZxwvgLnxNSZwFTDJJ49xVY5teaSvF7lzlo3YQfM=", + "lastModified": 1693163878, + "narHash": "sha256-HXuyMUVaRSoIA602jfFuYGXt6AMZ+WUxuvLq8iJmYTA=", "owner": "ipetkov", "repo": "crane", - "rev": "8b08e96c9af8c6e3a2b69af5a7fa168750fcf88e", + "rev": "43db881168bc65b568d36ceb614a0fc8b276191b", "type": "github" }, "original": { @@ -48,11 +48,11 @@ "systems": "systems" }, "locked": { - "lastModified": 1689068808, - "narHash": "sha256-6ixXo3wt24N/melDWjq70UuHQLxGV8jZvooRanIHXw0=", + "lastModified": 1692799911, + "narHash": "sha256-3eihraek4qL744EvQXsK1Ha6C3CR7nnT8X2qWap4RNk=", "owner": "numtide", "repo": "flake-utils", - "rev": "919d646de7be200f3bf08cb76ae1f09402b6f9b4", + "rev": "f9e7cf818399d17d347f847525c5a5a8032e4e44", "type": "github" }, "original": { @@ -78,11 +78,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1689008574, - "narHash": "sha256-VFMgyHDiqsGDkRg73alv6OdHJAqhybryWHv77bSCGIw=", + "lastModified": 1693158576, + "narHash": "sha256-aRTTXkYvhXosGx535iAFUaoFboUrZSYb1Ooih/auGp0=", "owner": "nixos", "repo": "nixpkgs", - "rev": "4a729ce4b1fe5ec4fffc71c67c96aa5184ebb462", + "rev": "a999c1cc0c9eb2095729d5aa03e0d8f7ed256780", "type": "github" }, "original": { @@ -111,11 +111,11 @@ ] }, "locked": { - "lastModified": 1689042658, - "narHash": "sha256-p7cQAFNt5kX19sZvK74CmY0nTrtujpZg6sZUiV1ntAk=", + "lastModified": 1693276701, + "narHash": "sha256-rz1vcG4UyxLTgJDHOnU/9Z0LthXi9o7YQxRV/m+LJ+U=", "owner": "oxalica", "repo": "rust-overlay", - "rev": "d7181bb2237035df17cab9295c95f987f5c527e6", + "rev": "29d5b70c1fa6e1f9f3aa8f69361e09e95c609ab4", "type": "github" }, "original": { diff --git a/flake.nix b/flake.nix index 1005f0a..2842afd 100644 --- a/flake.nix +++ b/flake.nix @@ -74,7 +74,7 @@ ]; }; - rustToolchain = pkgs.rust-bin.nightly."2023-05-22".default.override { + rustToolchain = pkgs.rust-bin.nightly."2023-08-28".default.override { extensions = [ "rustc" "cargo"