fix: fix a race condition in ar_coordinated_mining_tests #2174
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: "Arweave Tests" | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: ["**"] | |
| jobs: | |
| build: | |
| runs-on: self-hosted | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "recursive" | |
| # only arweave dependencies are being cached, | |
| # those are not updated everyday and this is | |
| # unecessary to fetch them everytime. | |
| - uses: actions/cache@v4 | |
| id: cache | |
| with: | |
| path: | | |
| _build/default/lib/accept | |
| _build/default/lib/b64fast | |
| _build/default/lib/cowboy | |
| _build/default/lib/cowlib | |
| _build/default/lib/graphql | |
| _build/default/lib/gun | |
| _build/default/lib/jiffy | |
| _build/default/lib/prometheus | |
| _build/default/lib/prometheus_cowboy | |
| _build/default/lib/prometheus_httpd | |
| _build/default/lib/prometheus_process_collector | |
| _build/default/lib/quantile_estimator | |
| _build/default/lib/ranch | |
| _build/default/lib/.rebar3 | |
| _build/default/lib/recon | |
| _build/default/lib/rocksdb | |
| _build/default/plugins/ | |
| _build/default/plugins/aleppo | |
| _build/default/plugins/geas | |
| _build/default/plugins/geas_rebar3 | |
| _build/default/plugins/hex_core | |
| _build/default/plugins/katana_code | |
| _build/default/plugins/pc | |
| _build/default/plugins/.rebar3 | |
| _build/default/plugins/rebar3_archive_plugin | |
| _build/default/plugins/rebar3_elvis_plugin | |
| _build/default/plugins/rebar3_hex | |
| _build/default/plugins/samovar | |
| _build/default/plugins/verl | |
| _build/default/plugins/zipper | |
| key: deps-cache-${{ hashFiles('rebar.lock') }} | |
| restore-keys: | | |
| deps-cache-${{ hashFiles('rebar.lock') }} | |
| - name: Get dependencies | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| run: ./ar-rebar3 test get-deps | |
| - uses: actions/cache@v4 | |
| if: steps.cache.outputs.cache-hit != 'true' | |
| with: | |
| path: | | |
| _build/default/lib/accept | |
| _build/default/lib/b64fast | |
| _build/default/lib/cowboy | |
| _build/default/lib/cowlib | |
| _build/default/lib/graphql | |
| _build/default/lib/gun | |
| _build/default/lib/jiffy | |
| _build/default/lib/prometheus | |
| _build/default/lib/prometheus_cowboy | |
| _build/default/lib/prometheus_httpd | |
| _build/default/lib/prometheus_process_collector | |
| _build/default/lib/quantile_estimator | |
| _build/default/lib/ranch | |
| _build/default/lib/.rebar3 | |
| _build/default/lib/recon | |
| _build/default/lib/rocksdb | |
| _build/default/plugins/ | |
| _build/default/plugins/aleppo | |
| _build/default/plugins/geas | |
| _build/default/plugins/geas_rebar3 | |
| _build/default/plugins/hex_core | |
| _build/default/plugins/katana_code | |
| _build/default/plugins/pc | |
| _build/default/plugins/.rebar3 | |
| _build/default/plugins/rebar3_archive_plugin | |
| _build/default/plugins/rebar3_elvis_plugin | |
| _build/default/plugins/rebar3_hex | |
| _build/default/plugins/samovar | |
| _build/default/plugins/verl | |
| _build/default/plugins/zipper | |
| key: deps-cache-${{ hashFiles('rebar.lock') }} | |
| - name: Build arweave test sources | |
| run: ./ar-rebar3 test compile | |
| # some artifacts are compiled and only available | |
| # in arweave directy (libraries) | |
| - name: Prepare artifacts | |
| run: | | |
| chmod -R u+w ./_build | |
| tar czfp _build.tar.gz ./_build | |
| tar czfp apps.tar.gz ./apps | |
| # to avoid reusing artifacts from someone else | |
| # and generating issues, an unique artifact is | |
| # produced using github checksum. | |
| - name: upload artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: build-${{ github.sha }} | |
| if-no-files-found: error | |
| retention-days: 1 | |
| overwrite: true | |
| path: | | |
| _build.tar.gz | |
| apps.tar.gz | |
| eunit-tests: | |
| needs: build | |
| runs-on: self-hosted | |
| strategy: | |
| fail-fast: true | |
| # 8 parallel jobs seem to be a correct value. 12 | |
| # jobs were used previously, but it was requiring | |
| # too much CPU and RAM (in particular if more than | |
| # one developer is used the pipelines). | |
| max-parallel: 8 | |
| matrix: | |
| core_test_mod: [ | |
| ## Long-running tests. Put these first to limit the overall runtime of the | |
| ## test suite | |
| ar_coordinated_mining_tests, | |
| ar_data_sync_tests, | |
| ar_fork_recovery_tests, | |
| ar_multiple_txs_per_wallet_tests, | |
| ar_packing_tests, | |
| ar_poa, | |
| ar_vdf_server_tests, | |
| ar_post_block_tests, | |
| ar_reject_chunks_tests, | |
| ## Modules containing tests | |
| ar, | |
| ar_block, | |
| ar_block_cache, | |
| ar_chain_stats, | |
| ar_chunk_storage, | |
| ar_data_sync_worker_master, | |
| ar_deep_hash, | |
| ar_diff_dag, | |
| ar_ets_intervals, | |
| ar_events, | |
| ar_inflation, | |
| ar_intervals, | |
| ar_join, | |
| ar_kv, | |
| ar_merkle, | |
| ar_mining_server, | |
| ar_mining_stats, | |
| ar_node, | |
| ar_node_utils, | |
| ar_nonce_limiter, | |
| # ar_p3, | |
| # ar_p3_config, | |
| # ar_p3_db, | |
| ar_packing_server, | |
| ar_patricia_tree, | |
| ar_peers, | |
| ar_pricing, | |
| ar_replica_2_9, | |
| ar_retarget, | |
| ar_serialize, | |
| ar_storage_module, | |
| ar_storage, | |
| ar_sync_buckets, | |
| ar_sync_record, | |
| ar_tx, | |
| ar_tx_db, | |
| ar_unbalanced_merkle, | |
| ar_util, | |
| ar_verify_chunks, | |
| ar_wallet, | |
| ar_webhook, | |
| ar_pool, | |
| ## Test modules (note: that _tests are implicitly run by a matching prefix name | |
| ar_base64_compatibility_tests, | |
| ar_config_tests, | |
| ar_difficulty_tests, | |
| ar_header_sync_tests, | |
| ar_http_iface_tests, | |
| ar_http_util_tests, | |
| ar_info_tests, | |
| ar_mempool_tests, | |
| ar_mine_randomx_tests, | |
| ar_mine_vdf_tests, | |
| ar_mining_io_tests, | |
| ar_poller_tests, | |
| ar_reject_chunks_tests, | |
| ar_semaphore_tests, | |
| ar_start_from_block_tests, | |
| ar_tx_blacklist_tests, | |
| ar_tx_replay_pool_tests, | |
| ar_vdf_tests, | |
| ] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| submodules: "recursive" | |
| - name: Download artifact | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: build-${{ github.sha }} | |
| # Both artifacts (_build and apps dir) are | |
| # required. | |
| - name: Extract artifact | |
| run: | | |
| tar zxfp _build.tar.gz | |
| tar zxfp apps.tar.gz | |
| - name: ${{ matrix.core_test_mod }}.erl | |
| id: tests | |
| run: | | |
| EXIT_CODE=0 | |
| export PATH=$(pwd)/_build/erts/bin:$PATH | |
| export ERL_EPMD_ADDRESS=127.0.0.1 | |
| export NAMESPACE="${{ matrix.core_test_mod }}" | |
| export ERL_PATH_ADD="$(echo $(pwd)/_build/test/lib/*/ebin)" | |
| export ERL_PATH_TEST="$(pwd)/_build/test/lib/arweave/test" | |
| export ERL_PATH_CONF="$(pwd)/config/sys.config" | |
| export ERL_TEST_OPTS="-pa ${ERL_PATH_ADD} ${ERL_PATH_TEST} -config ${ERL_PATH_CONF}" | |
| RETRYABLE=1 | |
| while [[ $RETRYABLE -eq 1 ]]; do | |
| RETRYABLE=0 | |
| set +e | |
| set -x | |
| erl +S 4:4 $ERL_TEST_OPTS -noshell -name "main-${NAMESPACE}@127.0.0.1" -setcookie "${{ matrix.core_test_mod }}" -run ar tests "${{ matrix.core_test_mod }}" -s init stop 2>&1 | tee main.out | |
| EXIT_CODE=${PIPESTATUS[0]} | |
| set +x | |
| set -e | |
| # For debugging purposes, print the peer1 output if the tests failed | |
| if [[ $EXIT_CODE -ne 0 ]]; then | |
| echo -e "\033[0;32m===> Checking for retry\033[0m" | |
| if ls peer1-*.out 1> /dev/null 2>&1; then | |
| first_line_peer1=$(head -n 1 peer1-*.out) | |
| fi | |
| first_line_main=$(head -n 1 main.out) | |
| echo -e "\033[0;31m===> First line of peer1 node's output: $first_line_peer1\033[0m" | |
| echo -e "\033[0;31m===> First line of main node's output: $first_line_main\033[0m" | |
| # Check if it is a retryable error | |
| if [[ "$first_line_peer1" == "Protocol 'inet_tcp': register/listen error: "* ]]; then | |
| echo "Retrying test because of inet_tcp error..." | |
| RETRYABLE=1 | |
| sleep 1 | |
| elif [[ "$first_line_peer1" == "Protocol 'inet_tcp': the name"* ]]; then | |
| echo "Retrying test because of inet_tcp clash..." | |
| RETRYABLE=1 | |
| sleep 1 | |
| elif [[ "$first_line_main" == *"econnrefused"* ]]; then | |
| echo "Retrying test because of econnrefused..." | |
| RETRYABLE=1 | |
| sleep 1 | |
| else | |
| if ls peer1-*.out 1> /dev/null 2>&1; then | |
| echo -e "\033[0;31m===> Test failed, printing the peer1 node's output...\033[0m" | |
| cat peer1-*.out | |
| else | |
| echo -e "\033[0;31m===> Test failed without peer1 output...\033[0m" | |
| fi | |
| if ls peer2-*.out 1> /dev/null 2>&1; then | |
| echo -e "\033[0;31m===> Test failed, printing the peer2 node's output...\033[0m" | |
| cat peer2-*.out | |
| else | |
| echo -e "\033[0;31m===> Test failed without peer2 output...\033[0m" | |
| fi | |
| if ls peer3-*.out 1> /dev/null 2>&1; then | |
| echo -e "\033[0;31m===> Test failed, printing the peer3 node's output...\033[0m" | |
| cat peer3-*.out | |
| else | |
| echo -e "\033[0;31m===> Test failed without peer3 output...\033[0m" | |
| fi | |
| if ls peer4-*.out 1> /dev/null 2>&1; then | |
| echo -e "\033[0;31m===> Test failed, printing the peer4 node's output...\033[0m" | |
| cat peer4-*.out | |
| else | |
| echo -e "\033[0;31m===> Test failed without peer4 output...\033[0m" | |
| fi | |
| fi | |
| fi | |
| done | |
| echo "exit_code=$EXIT_CODE" >> $GITHUB_ENV # Set the exit_code output variable using Environment Files | |
| exit $EXIT_CODE # exit with the exit code of the tests |