diff --git a/Cargo.lock b/Cargo.lock index f0b0e80f227..e78622f8edb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -494,18 +494,17 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "chrono" -version = "0.4.26" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +checksum = "7f2c685bad3eb3d45a01354cedb7d5faa66194d1d58ba6e267a8de788f79db38" dependencies = [ "android-tzdata", "iana-time-zone", "js-sys", "num-traits", "serde", - "time 0.1.44", "wasm-bindgen", - "winapi", + "windows-targets 0.48.0", ] [[package]] @@ -1527,7 +1526,7 @@ dependencies = [ "proc-macro2", "quote", "syn 1.0.107", - "time 0.3.17", + "time", ] [[package]] @@ -4470,17 +4469,6 @@ dependencies = [ "once_cell", ] -[[package]] -name = "time" -version = "0.1.44" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db9e6914ab8b1ae1c260a4ae7a49b6c5611b40328a735b21862567685e73255" -dependencies = [ - "libc", - "wasi 0.10.0+wasi-snapshot-preview1", - "winapi", -] - [[package]] name = "time" version = "0.3.17" @@ -5530,9 +5518,9 @@ dependencies = [ [[package]] name = "webpki" -version = "0.22.0" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f095d78192e208183081cc07bc5515ef55216397af48b873e5edcd72637fa1bd" +checksum = "f0e74f82d49d545ad128049b7e88f6576df2da6b02e9ce565c6f533be576957e" dependencies = [ "ring", "untrusted", diff --git a/NEWS.md b/NEWS.md index cbc2216577d..4298ed35999 100644 --- a/NEWS.md +++ b/NEWS.md @@ -12,7 +12,7 @@ - **Initialization handler** - A new block handler filter `once` for `ethereum` data sources which enables subgraph developers to create a handle which will be called only once before all other handlers run. This configuration allows the subgraph to use the handler as an initialization handler, performing specific tasks at the start of indexing. [(#4725)](https://github.com/graphprotocol/graph-node/pull/4725) - **DataSourceContext in manifest** - `DataSourceContext` in Manifest - DataSourceContext can now be defined in the subgraph manifest. It's a free-form map accessible from the mapping. This feature is useful for templating chain-specific data in subgraphs that use the same codebase across multiple chains.[(#4848)](https://github.com/graphprotocol/graph-node/pull/4848) - `graph-node` version in index node API - The Index Node API now features a new query, Version, which can be used to query the current graph-node version and commit. [(#4852)](https://github.com/graphprotocol/graph-node/pull/4852) -- Added subgraph status to index node API [(#4779)](https://github.com/graphprotocol/graph-node/pull/4779) +- Added a '`paused`' field to Index Node API, a boolean indicating the subgraph’s pause status. [(#4779)](https://github.com/graphprotocol/graph-node/pull/4779) - Proof of Indexing logs now include block number [(#4798)](https://github.com/graphprotocol/graph-node/pull/4798) - `subgraph_features` table now tracks details about handlers used in a subgraph [(#4820)](https://github.com/graphprotocol/graph-node/pull/4820) - Configurable SSL for Postgres in Dockerfile - ssl-mode for Postgres can now be configured via the connection string when deploying through Docker, offering enhanced flexibility in database security settings.[(#4840)](https://github.com/graphprotocol/graph-node/pull/4840) @@ -43,7 +43,7 @@ Not Relevant * Update docker-compose.yml by @computeronix in https://github.com/graphprotocol/graph-node/pull/4844 --> -**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.32.0...e253ee14cda2d8456a86ae8f4e3f74a1a7979953 +**Full Changelog**: https://github.com/graphprotocol/graph-node/compare/v0.33.0...e253ee14cda2d8456a86ae8f4e3f74a1a7979953 ## v0.32.0 diff --git a/graph/Cargo.toml b/graph/Cargo.toml index 4c535afb43c..60d7a31f251 100644 --- a/graph/Cargo.toml +++ b/graph/Cargo.toml @@ -13,7 +13,7 @@ bytes = "1.0.1" cid = "0.10.1" diesel = { version = "1.4.8", features = ["postgres", "serde_json", "numeric", "r2d2", "chrono"] } diesel_derives = "1.4" -chrono = "0.4.25" +chrono = "0.4.31" envconfig = "0.10.0" Inflector = "0.11.3" isatty = "0.1.9" diff --git a/graph/src/data/store/mod.rs b/graph/src/data/store/mod.rs index f77d59daed8..8a95030900f 100644 --- a/graph/src/data/store/mod.rs +++ b/graph/src/data/store/mod.rs @@ -663,9 +663,6 @@ impl>> TryIntoEntityIterator< #[derive(Debug, Error, PartialEq, Eq, Clone)] pub enum EntityValidationError { - #[error("The provided entity has fields not defined in the schema for entity `{entity}`")] - FieldsNotDefined { entity: String }, - #[error("Entity {entity}[{id}]: unknown entity type `{entity}`")] UnknownEntityType { entity: String, id: String }, @@ -928,14 +925,6 @@ impl Entity { } })?; - for field in self.0.atoms() { - if !schema.has_field(&key.entity_type, field) { - return Err(EntityValidationError::FieldsNotDefined { - entity: key.entity_type.clone().into_string(), - }); - } - } - for field in &object_type.fields { let is_derived = field.is_derived(); match (self.get(&field.name), is_derived) { diff --git a/graph/src/data/subgraph/api_version.rs b/graph/src/data/subgraph/api_version.rs index f4a62b512dd..5e642719d95 100644 --- a/graph/src/data/subgraph/api_version.rs +++ b/graph/src/data/subgraph/api_version.rs @@ -15,6 +15,9 @@ pub const API_VERSION_0_0_6: Version = Version::new(0, 0, 6); /// Enables event handlers to require transaction receipts in the runtime. pub const API_VERSION_0_0_7: Version = Version::new(0, 0, 7); +/// Enables validation for fields that doesnt exist in the schema for an entity. +pub const API_VERSION_0_0_8: Version = Version::new(0, 0, 8); + /// Before this check was introduced, there were already subgraphs in the wild with spec version /// 0.0.3, due to confusion with the api version. To avoid breaking those, we accept 0.0.3 though it /// doesn't exist. diff --git a/graph/src/env/mappings.rs b/graph/src/env/mappings.rs index 25c224bb229..6f7e5022ab3 100644 --- a/graph/src/env/mappings.rs +++ b/graph/src/env/mappings.rs @@ -16,7 +16,7 @@ pub struct EnvVarsMapping { /// kilobytes). The default value is 10 megabytes. pub entity_cache_size: usize, /// Set by the environment variable `GRAPH_MAX_API_VERSION`. The default - /// value is `0.0.7`. + /// value is `0.0.8`. pub max_api_version: Version, /// Set by the environment variable `GRAPH_MAPPING_HANDLER_TIMEOUT` /// (expressed in seconds). No default is provided. @@ -93,7 +93,7 @@ pub struct InnerMappingHandlers { entity_cache_dead_weight: EnvVarBoolean, #[envconfig(from = "GRAPH_ENTITY_CACHE_SIZE", default = "10000")] entity_cache_size_in_kb: usize, - #[envconfig(from = "GRAPH_MAX_API_VERSION", default = "0.0.7")] + #[envconfig(from = "GRAPH_MAX_API_VERSION", default = "0.0.8")] max_api_version: Version, #[envconfig(from = "GRAPH_MAPPING_HANDLER_TIMEOUT")] mapping_handler_timeout_in_secs: Option, diff --git a/graph/src/schema/input_schema.rs b/graph/src/schema/input_schema.rs index de26dd30149..872ad43082b 100644 --- a/graph/src/schema/input_schema.rs +++ b/graph/src/schema/input_schema.rs @@ -379,6 +379,15 @@ impl InputSchema { .map(|fields| fields.contains(&field)) .unwrap_or(false) } + + pub fn has_field_with_name(&self, entity_type: &EntityType, field: &str) -> bool { + let field = self.inner.pool.lookup(field); + + match field { + Some(field_atom) => self.has_field(entity_type, field_atom), + None => false, + } + } } /// Create a new pool that contains the names of all the types defined diff --git a/runtime/test/src/test.rs b/runtime/test/src/test.rs index 7b68bd41dfc..4f5204aa3ae 100644 --- a/runtime/test/src/test.rs +++ b/runtime/test/src/test.rs @@ -1225,8 +1225,13 @@ struct Host { } impl Host { - async fn new(schema: &str, deployment_hash: &str, wasm_file: &str) -> Host { - let version = ENV_VARS.mappings.max_api_version.clone(); + async fn new( + schema: &str, + deployment_hash: &str, + wasm_file: &str, + api_version: Option, + ) -> Host { + let version = api_version.unwrap_or(ENV_VARS.mappings.max_api_version.clone()); let wasm_file = wasm_file_path(wasm_file, API_VERSION_0_0_5); let ds = mock_data_source(&wasm_file, version.clone()); @@ -1324,7 +1329,7 @@ async fn test_store_set_id() { name: String, }"; - let mut host = Host::new(schema, "hostStoreSetId", "boolean.wasm").await; + let mut host = Host::new(schema, "hostStoreSetId", "boolean.wasm", None).await; host.store_set(USER, UID, vec![("id", "u1"), ("name", "user1")]) .expect("setting with same id works"); @@ -1414,7 +1419,13 @@ async fn test_store_set_invalid_fields() { test2: String }"; - let mut host = Host::new(schema, "hostStoreSetInvalidFields", "boolean.wasm").await; + let mut host = Host::new( + schema, + "hostStoreSetInvalidFields", + "boolean.wasm", + Some(API_VERSION_0_0_8), + ) + .await; host.store_set(USER, UID, vec![("id", "u1"), ("name", "user1")]) .unwrap(); @@ -1437,8 +1448,7 @@ async fn test_store_set_invalid_fields() { // So we just check the string contains them let err_string = err.to_string(); dbg!(err_string.as_str()); - assert!(err_string - .contains("The provided entity has fields not defined in the schema for entity `User`")); + assert!(err_string.contains("Attempted to set undefined fields [test, test2] for the entity type `User`. Make sure those fields are defined in the schema.")); let err = host .store_set( @@ -1449,8 +1459,30 @@ async fn test_store_set_invalid_fields() { .err() .unwrap(); - err_says( - err, - "Unknown key `test3`. It probably is not part of the schema", + err_says(err, "Attempted to set undefined fields [test3] for the entity type `User`. Make sure those fields are defined in the schema."); + + // For apiVersion below 0.0.8, we should not error out + let mut host2 = Host::new( + schema, + "hostStoreSetInvalidFields", + "boolean.wasm", + Some(API_VERSION_0_0_7), ) + .await; + + let err_is_none = host2 + .store_set( + USER, + UID, + vec![ + ("id", "u1"), + ("name", "user1"), + ("test", "invalid_field"), + ("test2", "invalid_field"), + ], + ) + .err() + .is_none(); + + assert!(err_is_none); } diff --git a/runtime/wasm/src/host_exports.rs b/runtime/wasm/src/host_exports.rs index a22d0a1376d..4e96723aab9 100644 --- a/runtime/wasm/src/host_exports.rs +++ b/runtime/wasm/src/host_exports.rs @@ -4,6 +4,7 @@ use std::ops::Deref; use std::str::FromStr; use std::time::{Duration, Instant}; +use graph::data::subgraph::API_VERSION_0_0_8; use graph::data::value::Word; use never::Never; @@ -151,6 +152,54 @@ impl HostExports { ))) } + fn check_invalid_fields( + &self, + api_version: Version, + data: &HashMap, + state: &BlockState, + entity_type: &EntityType, + ) -> Result<(), HostExportError> { + if api_version >= API_VERSION_0_0_8 { + let has_invalid_fields = data.iter().any(|(field_name, _)| { + !state + .entity_cache + .schema + .has_field_with_name(entity_type, &field_name) + }); + + if has_invalid_fields { + let mut invalid_fields: Vec = data + .iter() + .filter_map(|(field_name, _)| { + if !state + .entity_cache + .schema + .has_field_with_name(entity_type, &field_name) + { + Some(field_name.clone()) + } else { + None + } + }) + .collect(); + + invalid_fields.sort(); + + return Err(HostExportError::Deterministic(anyhow!( + "Attempted to set undefined fields [{}] for the entity type `{}`. Make sure those fields are defined in the schema.", + invalid_fields + .iter() + .map(|f| f.as_str()) + .collect::>() + .join(", "), + entity_type + ))); + } + } + + Ok(()) + } + pub(crate) fn store_set( &self, logger: &Logger, @@ -199,9 +248,19 @@ impl HostExports { } } + self.check_invalid_fields(self.api_version.clone(), &data, state, &key.entity_type)?; + + // Filter out fields that are not in the schema + let filtered_entity_data = data.into_iter().filter(|(field_name, _)| { + state + .entity_cache + .schema + .has_field_with_name(&key.entity_type, field_name) + }); + let entity = state .entity_cache - .make_entity(data.into_iter().map(|(key, value)| (key, value))) + .make_entity(filtered_entity_data) .map_err(|e| HostExportError::Deterministic(anyhow!(e)))?; let poi_section = stopwatch.start_section("host_export_store_set__proof_of_indexing"); diff --git a/tests/runner-tests/api-version/abis/Contract.abi b/tests/runner-tests/api-version/abis/Contract.abi new file mode 100644 index 00000000000..9d9f56b9263 --- /dev/null +++ b/tests/runner-tests/api-version/abis/Contract.abi @@ -0,0 +1,15 @@ +[ + { + "anonymous": false, + "inputs": [ + { + "indexed": false, + "internalType": "string", + "name": "testCommand", + "type": "string" + } + ], + "name": "TestEvent", + "type": "event" + } +] diff --git a/tests/runner-tests/api-version/data.0.0.7.json b/tests/runner-tests/api-version/data.0.0.7.json new file mode 100644 index 00000000000..d5496551483 --- /dev/null +++ b/tests/runner-tests/api-version/data.0.0.7.json @@ -0,0 +1,3 @@ +{ + "apiVersion": "0.0.7" +} diff --git a/tests/runner-tests/api-version/data.0.0.8.json b/tests/runner-tests/api-version/data.0.0.8.json new file mode 100644 index 00000000000..f01f6e94057 --- /dev/null +++ b/tests/runner-tests/api-version/data.0.0.8.json @@ -0,0 +1,3 @@ +{ + "apiVersion": "0.0.8" +} diff --git a/tests/runner-tests/api-version/package.json b/tests/runner-tests/api-version/package.json new file mode 100644 index 00000000000..503c7595204 --- /dev/null +++ b/tests/runner-tests/api-version/package.json @@ -0,0 +1,29 @@ +{ + "name": "api-version", + "version": "0.1.0", + "scripts": { + "build-contracts": "../../common/build-contracts.sh", + "codegen": "graph codegen --skip-migrations", + "test": "yarn build-contracts && truffle test --compile-none --network test", + "create:test": "graph create test/api-version --node $GRAPH_NODE_ADMIN_URI", + "prepare:0-0-7": "mustache data.0.0.7.json subgraph.template.yaml > subgraph.yaml", + "prepare:0-0-8": "mustache data.0.0.8.json subgraph.template.yaml > subgraph.yaml", + "deploy:test-0-0-7": "yarn prepare:0-0-7 && graph deploy test/api-version-0-0-7 --version-label 0.0.7 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI", + "deploy:test-0-0-8": "yarn prepare:0-0-8 && graph deploy test/api-version-0-0-8 --version-label 0.0.8 --ipfs $IPFS_URI --node $GRAPH_NODE_ADMIN_URI" + }, + "devDependencies": { + "@graphprotocol/graph-cli": "0.53.0", + "@graphprotocol/graph-ts": "0.31.0", + "solc": "^0.8.2" + }, + "dependencies": { + "@truffle/contract": "^4.3", + "@truffle/hdwallet-provider": "^1.2", + "apollo-fetch": "^0.7.0", + "babel-polyfill": "^6.26.0", + "babel-register": "^6.26.0", + "gluegun": "^4.6.1", + "mustache": "^4.2.0", + "truffle": "^5.2" + } +} diff --git a/tests/runner-tests/api-version/schema.graphql b/tests/runner-tests/api-version/schema.graphql new file mode 100644 index 00000000000..32db8d43674 --- /dev/null +++ b/tests/runner-tests/api-version/schema.graphql @@ -0,0 +1,4 @@ +type TestResult @entity { + id: ID! + message: String! +} diff --git a/tests/runner-tests/api-version/src/mapping.ts b/tests/runner-tests/api-version/src/mapping.ts new file mode 100644 index 00000000000..7a50ee868e6 --- /dev/null +++ b/tests/runner-tests/api-version/src/mapping.ts @@ -0,0 +1,15 @@ +import { Entity, Value, store } from "@graphprotocol/graph-ts"; +import { TestEvent } from "../generated/Contract/Contract"; +import { TestResult } from "../generated/schema"; + +export function handleTestEvent(event: TestEvent): void { + let testResult = new TestResult(event.params.testCommand); + testResult.message = event.params.testCommand; + let testResultEntity = testResult as Entity; + testResultEntity.set( + "invalid_field", + Value.fromString("This is an invalid field"), + ); + store.set("TestResult", testResult.id, testResult); + testResult.save(); +} diff --git a/tests/runner-tests/api-version/subgraph.template.yaml b/tests/runner-tests/api-version/subgraph.template.yaml new file mode 100644 index 00000000000..c1429c63b90 --- /dev/null +++ b/tests/runner-tests/api-version/subgraph.template.yaml @@ -0,0 +1,23 @@ +specVersion: 0.0.4 +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum/contract + name: Contract + network: test + source: + address: "0x0000000000000000000000000000000000000000" + abi: Contract + mapping: + kind: ethereum/events + apiVersion: {{apiVersion}} + language: wasm/assemblyscript + abis: + - name: Contract + file: ./abis/Contract.abi + entities: + - Call + eventHandlers: + - event: TestEvent(string) + handler: handleTestEvent + file: ./src/mapping.ts \ No newline at end of file diff --git a/tests/runner-tests/api-version/subgraph.yaml b/tests/runner-tests/api-version/subgraph.yaml new file mode 100644 index 00000000000..464a10d3f0c --- /dev/null +++ b/tests/runner-tests/api-version/subgraph.yaml @@ -0,0 +1,23 @@ +specVersion: 0.0.4 +schema: + file: ./schema.graphql +dataSources: + - kind: ethereum/contract + name: Contract + network: test + source: + address: "0x0000000000000000000000000000000000000000" + abi: Contract + mapping: + kind: ethereum/events + apiVersion: 0.0.8 + language: wasm/assemblyscript + abis: + - name: Contract + file: ./abis/Contract.abi + entities: + - Call + eventHandlers: + - event: TestEvent(string) + handler: handleTestEvent + file: ./src/mapping.ts \ No newline at end of file diff --git a/tests/tests/runner_tests.rs b/tests/tests/runner_tests.rs index be12c956929..7411e5f3176 100644 --- a/tests/tests/runner_tests.rs +++ b/tests/tests/runner_tests.rs @@ -41,7 +41,24 @@ impl RunnerTestRecipe { let (stores, hash) = tokio::join!( stores("./runner-tests/config.simple.toml"), - build_subgraph(&test_dir) + build_subgraph(&test_dir, None) + ); + + Self { + stores, + subgraph_name, + hash, + } + } + + /// Builds a new test subgraph with a custom deploy command. + async fn new_with_custom_cmd(subgraph_name: &str, deploy_cmd: &str) -> Self { + let subgraph_name = SubgraphName::new(subgraph_name).unwrap(); + let test_dir = format!("./runner-tests/{}", subgraph_name); + + let (stores, hash) = tokio::join!( + stores("./runner-tests/config.simple.toml"), + build_subgraph(&test_dir, Some(deploy_cmd)) ); Self { @@ -150,6 +167,80 @@ async fn typename() -> anyhow::Result<()> { Ok(()) } +#[tokio::test] +async fn api_version_0_0_7() { + let RunnerTestRecipe { + stores, + subgraph_name, + hash, + } = RunnerTestRecipe::new_with_custom_cmd("api-version", "deploy:test-0-0-7").await; + + // Before apiVersion 0.0.8 we allowed setting fields not defined in the schema. + // This test tests that it is still possible for lower apiVersion subgraphs + // to set fields not defined in the schema. + + let blocks = { + let block_0 = genesis(); + let mut block_1 = empty_block(block_0.ptr(), test_ptr(1)); + push_test_log(&mut block_1, "0.0.7"); + vec![block_0, block_1] + }; + + let stop_block = blocks.last().unwrap().block.ptr(); + + let chain = chain(blocks, &stores, None).await; + let ctx = fixture::setup(subgraph_name.clone(), &hash, &stores, &chain, None, None).await; + + ctx.start_and_sync_to(stop_block).await; + + let query_res = ctx + .query(&format!(r#"{{ testResults{{ id, message }} }}"#,)) + .await + .unwrap(); + + assert_json_eq!( + query_res, + Some(object! { + testResults: vec![ + object! { id: "0.0.7", message: "0.0.7" }, + ] + }) + ); +} + +#[tokio::test] +async fn api_version_0_0_8() { + let RunnerTestRecipe { + stores, + subgraph_name, + hash, + } = RunnerTestRecipe::new_with_custom_cmd("api-version", "deploy:test-0-0-8").await; + + // From apiVersion 0.0.8 we disallow setting fields not defined in the schema. + // This test tests that it is not possible to set fields not defined in the schema. + + let blocks = { + let block_0 = genesis(); + let mut block_1 = empty_block(block_0.ptr(), test_ptr(1)); + push_test_log(&mut block_1, "0.0.8"); + vec![block_0, block_1] + }; + + let chain = chain(blocks.clone(), &stores, None).await; + let ctx = fixture::setup(subgraph_name.clone(), &hash, &stores, &chain, None, None).await; + let stop_block = blocks.last().unwrap().block.ptr(); + let err = ctx.start_and_sync_to_error(stop_block.clone()).await; + let message = "transaction 0000000000000000000000000000000000000000000000000000000000000000: Attempted to set undefined fields [invalid_field] for the entity type `TestResult`. Make sure those fields are defined in the schema.\twasm backtrace:\t 0: 0x2ebc - !src/mapping/handleTestEvent\t in handler `handleTestEvent` at block #1 (0000000000000000000000000000000000000000000000000000000000000001)".to_string(); + let expected_err = SubgraphError { + subgraph_id: ctx.deployment.hash.clone(), + message, + block_ptr: Some(stop_block), + handler: None, + deterministic: true, + }; + assert_eq!(err, expected_err); +} + #[tokio::test] async fn derived_loaders() { let RunnerTestRecipe { @@ -954,8 +1045,11 @@ async fn poi_for_deterministically_failed_sg() -> anyhow::Result<()> { Ok(()) } -async fn build_subgraph(dir: &str) -> DeploymentHash { - build_subgraph_with_yarn_cmd(dir, "deploy:test").await + +/// deploy_cmd is the command to run to deploy the subgraph. If it is None, the +/// default `yarn deploy:test` is used. +async fn build_subgraph(dir: &str, deploy_cmd: Option<&str>) -> DeploymentHash { + build_subgraph_with_yarn_cmd(dir, deploy_cmd.unwrap_or("deploy:test")).await } async fn build_subgraph_with_yarn_cmd(dir: &str, yarn_cmd: &str) -> DeploymentHash {