From d24324617006534a287a9db09e6e5bacf3c760ee Mon Sep 17 00:00:00 2001 From: Peter Lebbing Date: Fri, 16 Feb 2024 10:14:16 +0100 Subject: [PATCH] GitLab CI: Simplify finding binaries (#2669) The `build` job creates an artifact that subsequent jobs download to run the tests. The jobs that actually run the tests cannot use `cabal list-bin` because the artifact doesn't include enough of the build data. We can simplify finding the binaries by having the artifact include symlinks to the binaries, created in the `build` job where `cabal list-bin` still works. The artifact included some bits that seem redundant since PR #2403, they have been removed. The minimum version of `cabal-install` that contains a properly working `list-bin` command is 3.8.1.0. Because of the fixed security issue, we instead update `cabal-install` to 3.10.2.0 for all GHC's in CI. GHC versions in CI are updated to the latest point release. Fixed a mistake in the Docker image: use the same Ubuntu version for building and for running. In GitLab CI, the `after_script` cannot use environment variables set in `before_script` or `script`. We use `zstd -T$THREADS` in `after_script`. Before this commit, that rendered to `zstd -T` which happens to mean "use all cores" and worked fine but was unintended. Triggered child pipelines (`.ci/gitlab/test.yml`) did not have `interruptible: true` set. Moved the `default` section into `.ci/gitlab/common.yml`. (cherry picked from commit b96bf079061b9a534b39c9ecb751195be05c657c) --- .ci/build.sh | 16 ++++++++++++ .ci/docker/Dockerfile | 4 +-- .ci/docker/build-and-publish-docker-image.sh | 6 +++-- .ci/find_cabal_bin.sh | 13 ---------- .ci/get_build_dist.sh | 10 ++++---- .ci/gitlab/benchmark.yml | 2 +- .ci/gitlab/common.yml | 14 ++++++++-- .ci/gitlab/test.yml | 27 ++++++++++---------- .gitlab-ci.yml | 13 ++-------- 9 files changed, 56 insertions(+), 49 deletions(-) delete mode 100755 .ci/find_cabal_bin.sh diff --git a/.ci/build.sh b/.ci/build.sh index b46d8e5637..2136b330f7 100755 --- a/.ci/build.sh +++ b/.ci/build.sh @@ -26,6 +26,22 @@ set -u # Build with default constraints cabal v2-build all --write-ghc-environment-files=always +# Put all the test binaries in a predictable location +TESTS=" +clash-cores:unittests +clash-cosim:test +clash-ffi:ffi-interface-tests +clash-lib:doctests +clash-lib:unittests +clash-prelude:doctests +clash-prelude:unittests +clash-testsuite:clash-testsuite +" +mkdir bin +for TEST in $TESTS; do + ln -s "$(realpath --relative-to=bin "$(cabal list-bin $TEST)")" bin/$TEST +done + # `CI_COMMIT_TAG` is set when a tag has been created on GitHub. We use this to # trigger a release pipeline (release to Snap / Hackage). if [[ ${CI_COMMIT_TAG:-} != "" ]]; then diff --git a/.ci/docker/Dockerfile b/.ci/docker/Dockerfile index a88cf957eb..f154b633da 100644 --- a/.ci/docker/Dockerfile +++ b/.ci/docker/Dockerfile @@ -6,7 +6,7 @@ # # To use buildkit, you need to set DOCKER_BUILDKIT=1 in your shell -ARG UBUNTU_VERSION=jammy-20230308 +ARG UBUNTU_VERSION FROM ubuntu:$UBUNTU_VERSION AS builder LABEL vendor="QBayLogic B.V." maintainer="devops@qbaylogic.com" @@ -122,7 +122,7 @@ RUN curl "https://downloads.haskell.org/~ghcup/$ghcup_version/x86_64-linux-ghcup && ghcup install ghc $ghc_version --set \ && ghcup install cabal $cabal_version --set -ARG UBUNTU_VERSION=focal-20210416 +ARG UBUNTU_VERSION FROM ubuntu:$UBUNTU_VERSION AS run LABEL vendor="QBayLogic B.V." maintainer="devops@qbaylogic.com" diff --git a/.ci/docker/build-and-publish-docker-image.sh b/.ci/docker/build-and-publish-docker-image.sh index 87c2618d1b..fbbf230c04 100755 --- a/.ci/docker/build-and-publish-docker-image.sh +++ b/.ci/docker/build-and-publish-docker-image.sh @@ -14,8 +14,9 @@ elif [[ "$1" != "" ]]; then exit 1 fi -GHC_VERSIONS=( "9.6.2" "9.4.6" "9.2.8" "9.0.2" "8.10.7" "8.8.4" "8.6.5") -CABAL_VERSIONS=("3.10.1.0" "3.8.1.0" "3.6.2.0" "3.4.0.0" "3.2.0.0" "3.2.0.0" "3.0.0.0") +UBUNTU_VERSION=jammy-20240125 +GHC_VERSIONS=( "9.6.4" "9.4.8" "9.2.8" "9.0.2" "8.10.7") +CABAL_VERSIONS=("3.10.2.0" "3.10.2.0" "3.10.2.0" "3.10.2.0" "3.10.2.0") # We want to use docker buildkit so that our layers are built in parallel. This # is ignored completely on versions of docker which don't support buildkit. @@ -27,6 +28,7 @@ do CABAL_VERSION="${CABAL_VERSIONS[i]}" docker build \ + --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} \ --build-arg cabal_version=${CABAL_VERSION} \ --build-arg ghc_version=${GHC_VERSION} \ -t "${REPO}/${NAME}${GHC_VERSION}:$now" \ diff --git a/.ci/find_cabal_bin.sh b/.ci/find_cabal_bin.sh deleted file mode 100755 index 1deedaeb5f..0000000000 --- a/.ci/find_cabal_bin.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -set -euo pipefail - -SIMPLE_PATH=$(echo dist-newstyle/build/*/*/"$1"-*/"$3"/"$2"/build/"$2"/"$2") -CUSTOM_PATH=$(echo dist-newstyle/build/*/*/"$1"-*/build/"$2"/"$2") - -if [ -x "$SIMPLE_PATH" ]; then - echo "$SIMPLE_PATH" -elif [ -x "$CUSTOM_PATH" ]; then - echo "$CUSTOM_PATH" -else - echo "find_cabal_bin.sh: Could not find $1 $2" >/dev/stderr -fi diff --git a/.ci/get_build_dist.sh b/.ci/get_build_dist.sh index f1f36ba3e1..e2cdf5f937 100755 --- a/.ci/get_build_dist.sh +++ b/.ci/get_build_dist.sh @@ -26,9 +26,9 @@ done echo "${GIT_ROOT}"/dist-newstyle ls "${GIT_ROOT}"/.ghc.environment.* -# Pack source distribution too to prevent rebuilds due to changed modification -# dates. -ls -d "${GIT_ROOT}"/clash-* +# Some build products for clash-cosim are here +echo "${GIT_ROOT}"/clash-cosim/src/cbits +echo "${GIT_ROOT}"/clash-cosim/src/prims -# Include compile options -echo "${GIT_ROOT}"/cabal.project.local +# Include symlinks to built binaries +echo "${GIT_ROOT}"/bin diff --git a/.ci/gitlab/benchmark.yml b/.ci/gitlab/benchmark.yml index 7185a14e13..76677b7e61 100644 --- a/.ci/gitlab/benchmark.yml +++ b/.ci/gitlab/benchmark.yml @@ -1,5 +1,5 @@ .benchmark: - image: ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2023-08-22 + image: ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2024-02-15 stage: test timeout: 2 hours variables: diff --git a/.ci/gitlab/common.yml b/.ci/gitlab/common.yml index c3f224e275..a47a811cad 100644 --- a/.ci/gitlab/common.yml +++ b/.ci/gitlab/common.yml @@ -1,11 +1,20 @@ +default: + # Make all tasks interruptible by default + interruptible: true + retry: + max: 2 + when: + - runner_system_failure + - stuck_or_timeout_failure + .common: - image: ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2023-08-22 + image: ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2024-02-15 timeout: 10 minutes stage: build variables: # Note that we copy+paste the image name into CACHE_FALLBACK_KEY. If we don't, # $GHC_VERSION gets inserted at verbatim, instead of resolving to some ghc version. - CACHE_FALLBACK_KEY: $CI_JOB_NAME-master-ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2023-08-22-2-3-non_protected + CACHE_FALLBACK_KEY: $CI_JOB_NAME-master-ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2024-02-15-2-3-non_protected GIT_SUBMODULE_STRATEGY: recursive TERM: xterm-color cache: @@ -21,6 +30,7 @@ - tar -xf cache.tar.zst -C / || true - .ci/setup.sh after_script: + - export THREADS=$(./.ci/effective_cpus.sh) - tar -cf - $(ls -d /root/.cabal /root/.stack || true) | zstd -T${THREADS} -3 > cache.tar.zst # We run tests on local machines if: diff --git a/.ci/gitlab/test.yml b/.ci/gitlab/test.yml index 37b6ee9e07..41fb9c968b 100644 --- a/.ci/gitlab/test.yml +++ b/.ci/gitlab/test.yml @@ -82,37 +82,37 @@ build: cores:unittests: extends: .test-nocache script: - - $(.ci/find_cabal_bin.sh clash-cores unittests t) --hide-successes + - bin/clash-cores:unittests --hide-successes cosim:unittests: extends: .test-nocache script: - - $(.ci/find_cabal_bin.sh clash-cosim test t) + - bin/clash-cosim:test prelude:unittests: extends: .test-nocache script: - - $(.ci/find_cabal_bin.sh clash-prelude unittests t) --hide-successes + - bin/clash-prelude:unittests --hide-successes lib:doctests: extends: .test-nocache script: - - $(.ci/find_cabal_bin.sh clash-lib doctests t) -j${THREADS} + - bin/clash-lib:doctests -j$THREADS lib:unittests: extends: .test-nocache script: - - $(.ci/find_cabal_bin.sh clash-lib unittests t) --hide-successes + - bin/clash-lib:unittests --hide-successes prelude:doctests: extends: .test-nocache script: - - $(.ci/find_cabal_bin.sh clash-prelude doctests t) -j${THREADS} + - bin/clash-prelude:doctests -j$THREADS ffi:interface-tests: extends: .test-nocache script: - - $(.ci/find_cabal_bin.sh clash-ffi ffi-interface-tests x) --smallcheck-max-count 2000 + - bin/clash-ffi:ffi-interface-tests --smallcheck-max-count 2000 # Tests run on local fast machines: @@ -128,17 +128,17 @@ build-clash-dev: suite:vhdl: extends: .test-cache-local script: - - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j${THREADS} -p .VHDL --hide-successes --no-vivado + - bin/clash-testsuite:clash-testsuite -j$THREADS -p .VHDL --hide-successes --no-vivado suite:verilog: extends: .test-cache-local script: - - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j${THREADS} -p .Verilog --hide-successes --no-vivado + - bin/clash-testsuite:clash-testsuite -j$THREADS -p .Verilog --hide-successes --no-vivado suite:systemverilog: extends: .test-cache-local script: - - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j${THREADS} -p .SystemVerilog --hide-successes --no-modelsim --no-vivado + - bin/clash-testsuite:clash-testsuite -j$THREADS -p .SystemVerilog --hide-successes --no-modelsim --no-vivado # Vivado is quite slow, so we only run a subset of the tests on development branches # with it. The full testsuite gets run with Vivado every night on 'master'. @@ -146,7 +146,7 @@ suite:cores: extends: .test-cache-local script: - source /opt/tools/Xilinx/Vivado/2022.1/settings64.sh - - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j$THREADS -p Cores --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys + - bin/clash-testsuite:clash-testsuite -j$THREADS -p Cores --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys tags: - local - vivado-2022.1-standard @@ -179,6 +179,7 @@ ffi:example: script: - cd clash-ffi/example && ./run-iverilog.sh after_script: + - export THREADS=$(./.ci/effective_cpus.sh) - tar -cf - /root/.cabal | zstd -T${THREADS} -3 > cache.tar.zst # Tests run on local fast machines with Vivado installed. We only run these at night @@ -195,7 +196,7 @@ suite:vivado:verilog: extends: .test-cache-local-nightly script: - source /opt/tools/Xilinx/Vivado/2022.1/settings64.sh - - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j$THREADS -p .Verilog --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys + - bin/clash-testsuite:clash-testsuite -j$THREADS -p .Verilog --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys tags: - local - vivado-2022.1-standard @@ -215,7 +216,7 @@ suite:vivado:verilog: # extends: .test-cache-local-nightly # script: # - source /opt/tools/Xilinx/Vivado/2022.1/settings64.sh -# - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j$THREADS -p .VHDL --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys +# - bin/clash-testsuite:clash-testsuite -j$THREADS -p .VHDL --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys # tags: # - local # - vivado-2022.1-standard diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 6799e23965..8556fa3545 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,12 +1,3 @@ -default: - # Make all tasks interruptible by default - interruptible: true - retry: - max: 2 - when: - - runner_system_failure - - stuck_or_timeout_failure - include: - '/.ci/gitlab/common.yml' - '/.ci/gitlab/publish.yml' @@ -39,10 +30,10 @@ tests: CI_PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE parallel: matrix: - - GHC_VERSION: 9.6.2 + - GHC_VERSION: 9.6.4 RUN_TESTS: "always" - - GHC_VERSION: [9.4.6, 9.2.8] + - GHC_VERSION: [9.4.8, 9.2.8] RUN_TESTS: "nightly" - GHC_VERSION: 9.0.2