fix: add chunk-prover docker and a script to generate full-proof for …

…batch tests (#283)

* Add chunk-prover Dockerfile, and a script to generate full-proof for batch tests.

* Upgrade to use `v0.9.5`.

README.md

@@ -70,6 +70,22 @@ Could specify arguments as
 cargo run --release --bin zkevm_verify -- --params=test_params --proof=proof_data
 ```
 
+### Scripts
+
+- If you have read access for DB, could run command to generate full-proof for batch tests:
+```
+export DB_HOST=
+export DB_USER=
+export DB_NAME=
+
+sh scripts/gen_full_chunk_proofs.sh BATCH_INDEX
+```
+
+### Dockers
+
+- `docker/chunk-prover` is used to build and run GPU chunk-prover.
+- `docker/mock-testnet` is used to build and run GPU mock-testnet (inner-prove or chunk-prove).
+
 ## License
 
 Licensed under either of

docker/chunk-prover/gpu/Dockerfile

@@ -0,0 +1,26 @@
+FROM scrolltech/cuda-go-rust-builder:cuda-11.7.1-go-1.19-rust-nightly-2022-12-10 as builder
+
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH
+WORKDIR /src
+ADD . /src
+RUN mkdir /.cargo && echo 'paths = ["/src/halo2-gpu/halo2_proofs"]' > /.cargo/config
+RUN cargo build --bin zkevm_prove --release
+Run cd ./target/release && find -name libzktrie.so | xargs -I {} cp {} ./
+RUN apt update && apt install -y curl
+RUN mkdir test_assets
+RUN curl -o ./test_assets/layer1.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/layer1.config
+RUN curl -o ./test_assets/layer2.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/layer2.config
+
+FROM nvidia/cuda:11.7.1-runtime-ubuntu22.04
+
+ENV LD_LIBRARY_PATH /usr/local/cuda/lib64:$LD_LIBRARY_PATH
+ENV SCROLL_PROVER_ASSETS_DIR /opt/test_assets
+ENV SCROLL_PROVER_PARAMS_DIR /opt/test_params
+ENV RUST_MIN_STACK 100000000
+ENV RUST_BACKTRACE 1
+ENV RUST_LOG trace
+WORKDIR /opt
+RUN mkdir -p /opt/test_assets
+COPY --from=builder /src/target/release/libzktrie.so /usr/local/lib/
+COPY --from=builder /src/target/release/zkevm_prove /bin/
+COPY --from=builder /src/test_assets/ /opt/test_assets/

docker/chunk-prover/gpu/docker-compose.yml

@@ -0,0 +1,12 @@
+version: '3'
+services:
+  chunk_prover:
+    image: {DOCKER_IMAGE}
+    runtime: nvidia
+    container_name: chunk-prover-gpu
+    environment:
+      - CHAIN_ID={CHAIN_ID}
+    volumes:
+      - {PARAMS_DIR}:/opt/test_params
+      - {CHUNK_DIR}:/opt/integration/test_chunk
+    command: zkevm_prove -t test_chunk

docker/mock-testnet/gpu/Dockerfile

@@ -9,13 +9,13 @@ RUN cargo build --features $MOCK_TESTNET_FEATURE --bin mock_testnet --release
 Run cd ./target/release && find -name libzktrie.so | xargs -I {} cp {} ./
 RUN apt update && apt install -y curl
 RUN mkdir test_assets
-RUN curl -o ./test_assets/layer1.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.1/layer1.config
-RUN curl -o ./test_assets/layer2.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.1/layer2.config
-RUN curl -o ./test_assets/layer3.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.1/layer3.config
-RUN curl -o ./test_assets/layer4.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.1/layer4.config
-RUN curl -o ./test_assets/agg_vk.vkey https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.1/agg_vk.vkey
-RUN curl -o ./test_assets/chunk.protocol https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.1/chunk.protocol
-RUN curl -o ./test_assets/evm_verifier.bin https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.1/evm_verifier.bin
+RUN curl -o ./test_assets/layer1.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/layer1.config
+RUN curl -o ./test_assets/layer2.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/layer2.config
+RUN curl -o ./test_assets/layer3.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/layer3.config
+RUN curl -o ./test_assets/layer4.config https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/layer4.config
+RUN curl -o ./test_assets/agg_vk.vkey https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/agg_vk.vkey
+RUN curl -o ./test_assets/chunk.protocol https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/chunk.protocol
+RUN curl -o ./test_assets/evm_verifier.bin https://circuit-release.s3.us-west-2.amazonaws.com/release-v0.9.5/evm_verifier.bin
 
 FROM nvidia/cuda:11.7.1-runtime-ubuntu22.04
 

docker/mock-testnet/gpu/docker-compose.yml

@@ -1,6 +1,6 @@
 version: '3'
 services:
-  prover:
+  mock_testnet:
     image: {DOCKER_IMAGE}
     runtime: nvidia
     container_name: mock-testnet-gpu

scripts/gen_full_chunk_proofs.sh

@@ -0,0 +1,55 @@
+set -e
+
+# The first command argument is batch-index.
+batch_index=$1
+if [[ ! $batch_index =~ ^[0-9]+$ ]]; then
+    echo "Must specify batch-index!"
+    exit
+fi
+
+# Set ENV DB_HOST, DB_USER and DB_NAME, and set password as:
+# https://www.postgresql.org/docs/current/libpq-pgpass.html
+db_host=$DB_HOST
+db_user=$DB_USER
+db_name=$DB_NAME
+if [ -z $db_host ] || [ -z $db_user ] || [ -z $db_name ]; then
+    echo "Must set ENV DB_HOST, DB_USER and DB_NAME!"
+    exit
+fi
+
+# Replace with ENV OUTPUT_FILE.
+output_file="${OUTPUT_FILE:-"full_proof_1.json"}"
+
+# Replace with ENV CHAIN_ID.
+chain_id="${CHAIN_ID:-534351}"
+
+# Get chunk-infos.
+chunk_infos=$(psql -h $db_host -U $db_user $db_name --csv -c "
+select json_agg(res) as infos from (
+    select
+        $chain_id as chain_id,
+        false as is_padding,
+        chunk.state_root              as post_state_root,
+        chunk.parent_chunk_state_root as prev_state_root,
+        chunk.withdraw_root           as withdraw_root,
+        chunk.hash                    as data_hash
+    from chunk join batch on chunk.batch_hash = batch.hash
+    where batch.index = $batch_index
+    order by chunk.index
+) res;")
+
+chunk_infos=$(echo $chunk_infos | sed 's/""/"/g')
+chunk_infos=$(echo $chunk_infos | sed 's/^infos "\(.*\)"$/"chunk_infos": \1/g')
+
+# Get chunk-proofs.
+chunk_proofs=$(psql -h $db_host -U $db_user $db_name --csv -c "
+select convert_from(chunk.proof, 'UTF-8') as proofs
+    from chunk join batch on chunk.batch_hash = batch.hash
+    where batch.index = $batch_index
+    order by chunk.index;")
+
+chunk_proofs=$(echo $chunk_proofs | sed 's/" "/,/g')
+chunk_proofs=$(echo $chunk_proofs | sed 's/""/"/g')
+chunk_proofs=$(echo $chunk_proofs | sed 's/^proofs "\(.*\)"$/"chunk_proofs": [\1]/g')
+
+echo "{$chunk_infos,$chunk_proofs}" | jq > $output_file