diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b6a102d08..1dd2f9f7b 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -10,9 +10,8 @@ on: - main env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - CARGO_INCREMENTAL: false + CARGO_NET_GIT_FETCH_WITH_CLI: true + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git jobs: skip_check: @@ -32,39 +31,37 @@ jobs: if: | github.event.pull_request.draft == false && (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true') - name: fmt + name: Rustfmt + timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: nightly-2022-12-10 - override: true - components: rustfmt, clippy - - uses: Swatinem/rust-cache@v2 - - name: Run cargo fmt - run: | - cargo build --release - cargo fmt --all -- --check + components: rustfmt + - name: Cargo cache + uses: Swatinem/rust-cache@v2 + - name: Cargo check + run: cargo check --all-features + - name: Cargo fmt + run: cargo fmt --all -- --check clippy: needs: [skip_check] if: | github.event.pull_request.draft == false && (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true') - name: clippy + name: Clippy + timeout-minutes: 30 runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 - - uses: actions-rs/toolchain@v1 + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master with: - profile: minimal toolchain: nightly-2022-12-10 - override: true - components: rustfmt, clippy - - uses: Swatinem/rust-cache@v2 - - name: Run cargo clippy - run: | - cargo build --release - cargo clippy --release --features prove_verify -- -D warnings + components: clippy + - name: Cargo cache + uses: Swatinem/rust-cache@v2 + - name: Run clippy + run: cargo clippy --all-features --all-targets -- -D warnings diff --git a/.github/workflows/unit_test.yml b/.github/workflows/unit_test.yml index 155a1301c..d5c7db705 100644 --- a/.github/workflows/unit_test.yml +++ b/.github/workflows/unit_test.yml @@ -10,9 +10,8 @@ on: - main env: - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - CARGO_INCREMENTAL: false + CARGO_NET_GIT_FETCH_WITH_CLI: true + CARGO_REGISTRIES_CRATES_IO_PROTOCOL: git jobs: skip_check: @@ -27,16 +26,15 @@ jobs: concurrent_skipping: 'same_content_newer' paths_ignore: '["**/README.md"]' - linux-test: + test: if: | github.event.pull_request.draft == false && (github.event.action == 'ready_for_review' || needs.skip_check.outputs.should_skip != 'true') - name: Linux Test on ${{ matrix.rust }} + name: Test runs-on: ubuntu-latest strategy: matrix: arch: [amd64] - rust: [nightly] container: image: ${{ matrix.arch }}/rust env: @@ -44,18 +42,32 @@ jobs: # "1" means line tables only, which is useful for panic tracebacks. RUSTFLAGS: "-C debuginfo=1" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: true - run: git config --system --add safe.directory /__w/scroll-prover/scroll-prover - - uses: actions/setup-go@v3 + - uses: dtolnay/rust-toolchain@master with: - go-version: '>=1.18.0' - - name: Setup Rust toolchain - run: | - rustup toolchain install ${{ matrix.rust }} - rustup default ${{ matrix.rust }} - rustup component add rustfmt - - uses: Swatinem/rust-cache@v2 + toolchain: nightly-2022-12-10 + - name: Cargo cache + uses: Swatinem/rust-cache@v2 + - name: Setup golang + uses: actions/setup-go@v3 + with: + cache: false + go-version: ~1.19 + # Go cache for building geth-utils + - name: Go cache + uses: actions/cache@v3 + with: + path: | + ~/.cache/go-build + ~/go/pkg/mod + key: ${{ matrix.arch }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ matrix.arch }}-go- + # https://github.com/actions/cache/issues/810 + env: + SEGMENT_DOWNLOAD_TIMEOUT_MINS: 5 - name: Run tests - run: cargo test + run: cargo test --verbose diff --git a/Cargo.lock b/Cargo.lock index d760e03c0..ac760b50e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -222,11 +222,10 @@ checksum = "d86b93f97252c47b41663388e6d155714a9d0c398b99f1005cbc5f978b29f445" [[package]] name = "bin" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "clap", - "dotenv", "ethers-providers 1.0.2", "integration", "itertools", @@ -864,12 +863,6 @@ dependencies = [ "subtle", ] -[[package]] -name = "dotenv" -version = "0.15.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77c90badedccf4105eca100756a0b1289e191f6fcbdadd3cee1d2f614f97da8f" - [[package]] name = "dotenvy" version = "0.15.7" @@ -1905,7 +1898,7 @@ dependencies = [ "httpdate", "itoa", "pin-project-lite", - "socket2", + "socket2 0.4.9", "tokio", "tower-service", "tracing", @@ -2037,7 +2030,7 @@ dependencies = [ [[package]] name = "integration" -version = "0.9.0" +version = "0.10.0" dependencies = [ "anyhow", "glob", @@ -2723,9 +2716,9 @@ dependencies = [ [[package]] name = "pin-project-lite" -version = "0.2.9" +version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" [[package]] name = "pin-utils" @@ -3720,6 +3713,16 @@ dependencies = [ "winapi", ] +[[package]] +name = "socket2" +version = "0.5.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" +dependencies = [ + "libc", + "windows-sys 0.48.0", +] + [[package]] name = "spin" version = "0.5.2" @@ -3936,11 +3939,10 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.29.1" +version = "1.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "532826ff75199d5833b9d2c5fe410f29235e25704ee5f0ef599fb51c21f4a4da" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ - "autocfg", "backtrace", "bytes", "libc", @@ -3949,7 +3951,7 @@ dependencies = [ "parking_lot 0.12.1", "pin-project-lite", "signal-hook-registry", - "socket2", + "socket2 0.5.4", "tokio-macros", "windows-sys 0.48.0", ] diff --git a/Cargo.toml b/Cargo.toml index 4011a6944..37a52ce7c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,6 +4,33 @@ members = [ "integration", ] +[workspace.package] +version = "0.10.0" +edition = "2021" +license = "MIT OR Apache-2.0" + +[workspace.dependencies] +anyhow = "1.0" +clap = { version = "3.1", features = ["derive"] } +ethers-providers = "1.0" +glob = "0.3" +itertools = "0.10" +log = "0.4" +log4rs = { version = "1.2", default_features = false, features = ["console_appender", "file_appender"] } +rand = "0.8" +rand_xorshift = "0.3" +reqwest = { version = "0.11", default-features = false, features = [ "json", "rustls-tls" ] } +serde = "1.0" +serde_derive = "1.0" +serde_json = "1.0" +tokio = { version = "1.32", features = ["full"] } + +halo2_proofs = { git = "https://github.com/scroll-tech/halo2.git", branch = "develop" } +prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] } +zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] } + +integration = { path = "integration" } + [patch.crates-io] ethers-core = { git = "https://github.com/scroll-tech/ethers-rs.git", branch = "v2.0.7" } [patch."https://github.com/privacy-scaling-explorations/halo2.git"] diff --git a/Makefile b/Makefile index e606d25fd..bad2321a9 100644 --- a/Makefile +++ b/Makefile @@ -13,11 +13,11 @@ build-release: ## Check build in release mode @cargo build --release fmt: ## Check whether the code is formatted correctly - @cargo fmt --all -- --check + @cargo check --all-features clippy: ## Run clippy checks over all workspace members @cargo check --all-features - @cargo clippy --release --features prove_verify -- -D warnings + @cargo clippy --all-features --all-targets -- -D warnings test: ## Run tests for all the workspace members @cargo test --release --all diff --git a/README.md b/README.md index 014d62754..8084530f2 100644 --- a/README.md +++ b/README.md @@ -29,19 +29,22 @@ make download-setup -e degree=DEGREE params_dir=PARAMS_DIR `make test-chunk-prove` and `make test-agg-prove` are the main testing entries for multi-level circuit constraint system of scroll-prover. Developers could understand how the system works by reading the codes of these tests. -Besides it, `make test-inner-prove` could be used to test the first-level circuit, and `make test-batch-prove` could be used to test the final two levels. +And there are other tests: +- `make test-inner-prove` could be used to test the first-level circuit. +- `make test-batch-prove` could be used to test the final two levels. +- `make test-batches-with-each-chunk-num-prove` could be used to test batch proving with different chunk numbers. ### Binaries -This repository is designed to be used as a Rust crate, rather than a standalone running process. However, you can still use the following command to run binaries locally. +Could use the following command to run binaries locally. -If you run into linking issues you may need to run +If run into linking issues you may need to run ```shell cp `find ./target/release/ | grep libzktrie.so` /usr/local/lib/ ``` To move the zktrielib into a path where your linker could locate it. -Run zkevm prover to generate chunk proof (the word-dir is `./integration`) +Run zkevm prover to generate chunk proof (work directory is `./integration`) ```shell cargo build --release --bin zkevm_prove @@ -53,10 +56,10 @@ Could specify arguments as export OUTPUT_DIR="proof_data" # Params file should be located in `./integration/test_params`. -cargo run --release --bin zkevm_prove -- --params=test_params --trace=tests/traces/erc20/10_transfer.json +cargo run --release --bin zkevm_prove -- --params=test_params --trace=tests/extra_traces/new.json ``` -Run zkevm verifier to verify chunk proof (the word-dir is `./integration`) +Run zkevm verifier to verify chunk proof (work directory is `./integration`) ```shell cargo build --release --bin zkevm_verify diff --git a/bin/Cargo.toml b/bin/Cargo.toml index 5646f9ed1..5e698eb74 100644 --- a/bin/Cargo.toml +++ b/bin/Cargo.toml @@ -1,27 +1,26 @@ [package] name = "bin" -version = "0.9.0" -edition = "2021" - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +version.workspace = true +edition.workspace = true +license.workspace = true [dependencies] -anyhow = "1.0" -clap = { version = "3.1.3", features = ["derive"] } -dotenv = "0.15.0" -ethers-providers = "1.0" -itertools = "0.10.5" -log = "0.4" -log4rs = { version = "1.2.0", default_features = false, features = ["console_appender", "file_appender"] } -rand = "0.8" -rand_xorshift = "0.3" -reqwest = { version = "0.11", default-features = false, features = [ "json", "rustls-tls" ] } -serde = "1.0" -serde_derive = "1.0" -serde_json = "1.0.66" -tokio = { version = "1", features = ["full"] } -integration = { path = "../integration" } -prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] } +anyhow.workspace = true +clap.workspace = true +ethers-providers.workspace = true +itertools.workspace = true +log.workspace = true +log4rs.workspace = true +rand.workspace = true +rand_xorshift.workspace = true +reqwest.workspace = true +serde.workspace = true +serde_derive.workspace = true +serde_json.workspace = true +tokio.workspace = true + +prover.workspace = true +integration.workspace = true [[bin]] name = "zkevm_prove" diff --git a/bin/src/prove.rs b/bin/src/prove.rs index c396b47e2..7a4cb267f 100644 --- a/bin/src/prove.rs +++ b/bin/src/prove.rs @@ -29,6 +29,7 @@ pub fn prove_batch(id: &str, chunk_proofs: Vec) { pub fn prove_chunk(id: &str, witness_block: &WitnessBlock) -> Option { let result = catch_unwind(AssertUnwindSafe(|| { + #[cfg(not(feature = "chunk-prove"))] let proof = None::; #[cfg(feature = "inner-prove")] diff --git a/integration/Cargo.toml b/integration/Cargo.toml index 4c447968e..1c6ae8559 100644 --- a/integration/Cargo.toml +++ b/integration/Cargo.toml @@ -1,23 +1,24 @@ [package] name = "integration" -version = "0.9.0" -edition = "2021" +version.workspace = true +edition.workspace = true +license.workspace = true [dependencies] -halo2_proofs = { git = "https://github.com/privacy-scaling-explorations/halo2.git", tag = "v2023_02_02" } -prover = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] } -zkevm-circuits = { git = "https://github.com/scroll-tech/zkevm-circuits.git", branch = "develop", default-features = false, features = ["parallel_syn", "scroll", "shanghai"] } +anyhow.workspace = true +glob.workspace = true +itertools.workspace = true +log.workspace = true +log4rs.workspace = true +rand.workspace = true -anyhow = "1.0" -glob = "0.3.0" -itertools = "0.10.5" -log = "0.4" -log4rs = { version = "1.2.0", default_features = false, features = ["console_appender", "file_appender"] } -rand = "0.8" +halo2_proofs.workspace = true +prover.workspace = true +zkevm-circuits.workspace = true [dev-dependencies] -serde = "1.0" -serde_derive = "1.0" +serde.workspace = true +serde_derive.workspace = true [features] default = [] diff --git a/integration/tests/aggregation_tests.rs b/integration/tests/aggregation_tests.rs index b104eb3fd..96d28c00c 100644 --- a/integration/tests/aggregation_tests.rs +++ b/integration/tests/aggregation_tests.rs @@ -58,7 +58,7 @@ fn gen_chunk_hashes_and_proofs( fn new_batch_prover(assets_dir: &str) -> Prover { env::set_var("AGG_VK_FILENAME", "vk_batch_agg.vkey"); env::set_var("CHUNK_PROTOCOL_FILENAME", "chunk_chunk_0.protocol"); - let prover = Prover::from_dirs(PARAMS_DIR, &assets_dir); + let prover = Prover::from_dirs(PARAMS_DIR, assets_dir); log::info!("Constructed batch prover"); prover @@ -71,8 +71,8 @@ fn prove_and_verify_batch( ) { // Load or generate aggregation snark (layer-3). let layer3_snark = batch_prover - .load_or_gen_last_agg_snark("agg", chunk_hashes_proofs, Some(&output_dir)) + .load_or_gen_last_agg_snark("agg", chunk_hashes_proofs, Some(output_dir)) .unwrap(); - gen_and_verify_batch_proofs(batch_prover, layer3_snark, &output_dir); + gen_and_verify_batch_proofs(batch_prover, layer3_snark, output_dir); } diff --git a/integration/tests/batch_tests.rs b/integration/tests/batch_tests.rs index 4810d14e1..7430e7faf 100644 --- a/integration/tests/batch_tests.rs +++ b/integration/tests/batch_tests.rs @@ -32,7 +32,7 @@ fn test_batches_with_each_chunk_num_prove_verify() { fs::create_dir_all(&output_dir).unwrap(); prove_and_verify_batch( - &output_dir.to_string_lossy().to_string(), + &output_dir.to_string_lossy(), &mut batch_prover, chunk_hashes_proofs[..=i].to_vec(), ); @@ -54,9 +54,9 @@ fn load_chunk_hashes_and_proofs( let chunk_proofs = batch_task_detail.chunk_proofs; let chunk_hashes_proofs: Vec<_> = chunk_hashes[..] - .to_vec() - .into_iter() - .zip(chunk_proofs[..].to_vec().into_iter()) + .iter() + .copied() + .zip(chunk_proofs[..].iter().cloned()) .collect(); // Dump chunk-procotol for further batch-proving. @@ -64,7 +64,7 @@ fn load_chunk_hashes_and_proofs( .first() .unwrap() .1 - .dump(&output_dir, "0") + .dump(output_dir, "0") .unwrap(); log::info!( @@ -77,7 +77,7 @@ fn load_chunk_hashes_and_proofs( fn new_batch_prover(assets_dir: &str) -> Prover { env::set_var("AGG_VK_FILENAME", "vk_batch_agg.vkey"); env::set_var("CHUNK_PROTOCOL_FILENAME", "chunk_chunk_0.protocol"); - let prover = Prover::from_dirs(PARAMS_DIR, &assets_dir); + let prover = Prover::from_dirs(PARAMS_DIR, assets_dir); log::info!("Constructed batch prover"); prover @@ -93,10 +93,10 @@ fn prove_and_verify_batch( // Load or generate aggregation snark (layer-3). let layer3_snark = batch_prover - .load_or_gen_last_agg_snark("agg", chunk_hashes_proofs, Some(&output_dir)) + .load_or_gen_last_agg_snark("agg", chunk_hashes_proofs, Some(output_dir)) .unwrap(); - gen_and_verify_batch_proofs(batch_prover, layer3_snark, &output_dir); + gen_and_verify_batch_proofs(batch_prover, layer3_snark, output_dir); log::info!("Prove batch END: chunk_num = {chunk_num}"); }