GitLab CI: Simplify finding binaries (#2669)

The `build` job creates an artifact that subsequent jobs download to run the tests. The jobs that actually run the tests cannot use `cabal list-bin` because the artifact doesn't include enough of the build data. We can simplify finding the binaries by having the artifact include symlinks to the binaries, created in the `build` job where `cabal list-bin` still works. The artifact included some bits that seem redundant since PR #2403, they have been removed. The minimum version of `cabal-install` that contains a properly working `list-bin` command is 3.8.1.0. Because of the fixed security issue, we instead update `cabal-install` to 3.10.2.0 for all GHC's in CI. GHC versions in CI are updated to the latest point release. Fixed a mistake in the Docker image: use the same Ubuntu version for building and for running. In GitLab CI, the `after_script` cannot use environment variables set in `before_script` or `script`. We use `zstd -T$THREADS` in `after_script`. Before this commit, that rendered to `zstd -T` which happens to mean "use all cores" and worked fine but was unintended. Triggered child pipelines (`.ci/gitlab/test.yml`) did not have `interruptible: true` set. Moved the `default` section into `.ci/gitlab/common.yml`. (cherry picked from commit b96bf07)
clash-lang · Jul 16, 2024 · d243246 · d243246
1 parent f1ada2e
commit d243246
Show file tree

Hide file tree

Showing 9 changed files with 56 additions and 49 deletions.
diff --git a/.ci/build.sh b/.ci/build.sh
@@ -26,6 +26,22 @@ set -u
 # Build with default constraints
 cabal v2-build all --write-ghc-environment-files=always
 
+# Put all the test binaries in a predictable location
+TESTS="
+clash-cores:unittests
+clash-cosim:test
+clash-ffi:ffi-interface-tests
+clash-lib:doctests
+clash-lib:unittests
+clash-prelude:doctests
+clash-prelude:unittests
+clash-testsuite:clash-testsuite
+"
+mkdir bin
+for TEST in $TESTS; do
+  ln -s "$(realpath --relative-to=bin "$(cabal list-bin $TEST)")" bin/$TEST
+done
+
 # `CI_COMMIT_TAG` is set when a tag has been created on GitHub. We use this to
 # trigger a release pipeline (release to Snap / Hackage).
 if [[ ${CI_COMMIT_TAG:-} != "" ]]; then

diff --git a/.ci/docker/Dockerfile b/.ci/docker/Dockerfile
@@ -6,7 +6,7 @@
 #
 # To use buildkit, you need to set DOCKER_BUILDKIT=1 in your shell
 
-ARG UBUNTU_VERSION=jammy-20230308
+ARG UBUNTU_VERSION
 FROM ubuntu:$UBUNTU_VERSION AS builder
 
 LABEL vendor="QBayLogic B.V." maintainer="devops@qbaylogic.com"
@@ -122,7 +122,7 @@ RUN curl "https://downloads.haskell.org/~ghcup/$ghcup_version/x86_64-linux-ghcup
  && ghcup install ghc $ghc_version --set \
  && ghcup install cabal $cabal_version --set
 
-ARG UBUNTU_VERSION=focal-20210416
+ARG UBUNTU_VERSION
 FROM ubuntu:$UBUNTU_VERSION AS run
 
 LABEL vendor="QBayLogic B.V." maintainer="devops@qbaylogic.com"

diff --git a/.ci/docker/build-and-publish-docker-image.sh b/.ci/docker/build-and-publish-docker-image.sh
@@ -14,8 +14,9 @@ elif [[ "$1" != "" ]]; then
   exit 1
 fi
 
-GHC_VERSIONS=(  "9.6.2"  "9.4.6"  "9.2.8"   "9.0.2"   "8.10.7"  "8.8.4"   "8.6.5")
-CABAL_VERSIONS=("3.10.1.0" "3.8.1.0" "3.6.2.0" "3.4.0.0" "3.2.0.0" "3.2.0.0" "3.0.0.0")
+UBUNTU_VERSION=jammy-20240125
+GHC_VERSIONS=(  "9.6.4"  "9.4.8"  "9.2.8"   "9.0.2"   "8.10.7")
+CABAL_VERSIONS=("3.10.2.0" "3.10.2.0" "3.10.2.0" "3.10.2.0" "3.10.2.0")
 
 # We want to use docker buildkit so that our layers are built in parallel. This
 # is ignored completely on versions of docker which don't support buildkit.
@@ -27,6 +28,7 @@ do
   CABAL_VERSION="${CABAL_VERSIONS[i]}"
 
   docker build \
+    --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} \
     --build-arg cabal_version=${CABAL_VERSION} \
     --build-arg ghc_version=${GHC_VERSION} \
     -t "${REPO}/${NAME}${GHC_VERSION}:$now" \

diff --git a/.ci/find_cabal_bin.sh b/.ci/find_cabal_bin.sh
diff --git a/.ci/get_build_dist.sh b/.ci/get_build_dist.sh
@@ -26,9 +26,9 @@ done
 echo "${GIT_ROOT}"/dist-newstyle
 ls "${GIT_ROOT}"/.ghc.environment.*
 
-# Pack source distribution too to prevent rebuilds due to changed modification
-# dates.
-ls -d "${GIT_ROOT}"/clash-*
+# Some build products for clash-cosim are here
+echo "${GIT_ROOT}"/clash-cosim/src/cbits
+echo "${GIT_ROOT}"/clash-cosim/src/prims
 
-# Include compile options
-echo "${GIT_ROOT}"/cabal.project.local
+# Include symlinks to built binaries
+echo "${GIT_ROOT}"/bin
diff --git a/.ci/gitlab/benchmark.yml b/.ci/gitlab/benchmark.yml
@@ -1,5 +1,5 @@
 .benchmark:
-  image: ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2023-08-22
+  image: ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2024-02-15
   stage: test
   timeout: 2 hours
   variables:

diff --git a/.ci/gitlab/common.yml b/.ci/gitlab/common.yml
@@ -1,11 +1,20 @@
+default:
+  # Make all tasks interruptible by default
+  interruptible: true
+  retry:
+    max: 2
+    when:
+      - runner_system_failure
+      - stuck_or_timeout_failure
+
 .common:
-  image: ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2023-08-22
+  image: ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2024-02-15
   timeout: 10 minutes
   stage: build
   variables:
     # Note that we copy+paste the image name into CACHE_FALLBACK_KEY. If we don't,
     # $GHC_VERSION gets inserted at verbatim, instead of resolving to some ghc version.
-    CACHE_FALLBACK_KEY: $CI_JOB_NAME-master-ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2023-08-22-2-3-non_protected
+    CACHE_FALLBACK_KEY: $CI_JOB_NAME-master-ghcr.io/clash-lang/clash-ci-$GHC_VERSION:2024-02-15-2-3-non_protected
     GIT_SUBMODULE_STRATEGY: recursive
     TERM: xterm-color
   cache:
@@ -21,6 +30,7 @@
     - tar -xf cache.tar.zst -C / || true
     - .ci/setup.sh
   after_script:
+    - export THREADS=$(./.ci/effective_cpus.sh)
     - tar -cf - $(ls -d /root/.cabal /root/.stack || true) | zstd -T${THREADS} -3 > cache.tar.zst
 
 # We run tests on local machines if:

diff --git a/.ci/gitlab/test.yml b/.ci/gitlab/test.yml
@@ -82,37 +82,37 @@ build:
 cores:unittests:
   extends: .test-nocache
   script:
-    - $(.ci/find_cabal_bin.sh clash-cores unittests t) --hide-successes
+    - bin/clash-cores:unittests --hide-successes
 
 cosim:unittests:
   extends: .test-nocache
   script:
-    - $(.ci/find_cabal_bin.sh clash-cosim test t)
+    - bin/clash-cosim:test
 
 prelude:unittests:
   extends: .test-nocache
   script:
-    - $(.ci/find_cabal_bin.sh clash-prelude unittests t) --hide-successes
+    - bin/clash-prelude:unittests --hide-successes
 
 lib:doctests:
   extends: .test-nocache
   script:
-    - $(.ci/find_cabal_bin.sh clash-lib doctests t) -j${THREADS}
+    - bin/clash-lib:doctests -j$THREADS
 
 lib:unittests:
   extends: .test-nocache
   script:
-    - $(.ci/find_cabal_bin.sh clash-lib unittests t) --hide-successes
+    - bin/clash-lib:unittests --hide-successes
 
 prelude:doctests:
   extends: .test-nocache
   script:
-    - $(.ci/find_cabal_bin.sh clash-prelude doctests t) -j${THREADS}
+    - bin/clash-prelude:doctests -j$THREADS
 
 ffi:interface-tests:
   extends: .test-nocache
   script:
-    - $(.ci/find_cabal_bin.sh clash-ffi ffi-interface-tests x) --smallcheck-max-count 2000
+    - bin/clash-ffi:ffi-interface-tests --smallcheck-max-count 2000
 
 # Tests run on local fast machines:
 
@@ -128,25 +128,25 @@ build-clash-dev:
 suite:vhdl:
   extends: .test-cache-local
   script:
-    - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j${THREADS} -p .VHDL --hide-successes --no-vivado
+    - bin/clash-testsuite:clash-testsuite -j$THREADS -p .VHDL --hide-successes --no-vivado
 
 suite:verilog:
   extends: .test-cache-local
   script:
-    - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j${THREADS} -p .Verilog --hide-successes --no-vivado
+    - bin/clash-testsuite:clash-testsuite -j$THREADS -p .Verilog --hide-successes --no-vivado
 
 suite:systemverilog:
   extends: .test-cache-local
   script:
-    - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j${THREADS} -p .SystemVerilog --hide-successes --no-modelsim --no-vivado
+    - bin/clash-testsuite:clash-testsuite -j$THREADS -p .SystemVerilog --hide-successes --no-modelsim --no-vivado
 
 # Vivado is quite slow, so we only run a subset of the tests on development branches
 # with it. The full testsuite gets run with Vivado every night on 'master'.
 suite:cores:
   extends: .test-cache-local
   script:
     - source /opt/tools/Xilinx/Vivado/2022.1/settings64.sh
-    - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j$THREADS -p Cores --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys
+    - bin/clash-testsuite:clash-testsuite -j$THREADS -p Cores --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys
   tags:
     - local
     - vivado-2022.1-standard
@@ -179,6 +179,7 @@ ffi:example:
   script:
      - cd clash-ffi/example && ./run-iverilog.sh
   after_script:
+    - export THREADS=$(./.ci/effective_cpus.sh)
     - tar -cf - /root/.cabal | zstd -T${THREADS} -3 > cache.tar.zst
 
 # Tests run on local fast machines with Vivado installed. We only run these at night
@@ -195,7 +196,7 @@ suite:vivado:verilog:
   extends: .test-cache-local-nightly
   script:
     - source /opt/tools/Xilinx/Vivado/2022.1/settings64.sh
-    - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j$THREADS -p .Verilog --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys
+    - bin/clash-testsuite:clash-testsuite -j$THREADS -p .Verilog --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys
   tags:
     - local
     - vivado-2022.1-standard
@@ -215,7 +216,7 @@ suite:vivado:verilog:
 #   extends: .test-cache-local-nightly
 #   script:
 #     - source /opt/tools/Xilinx/Vivado/2022.1/settings64.sh
-#     - $(.ci/find_cabal_bin.sh clash-testsuite clash-testsuite x) -j$THREADS -p .VHDL --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys
+#     - bin/clash-testsuite:clash-testsuite -j$THREADS -p .VHDL --hide-successes --no-modelsim --no-ghdl --no-iverilog --no-verilator --no-symbiyosys
 #   tags:
 #     - local
 #     - vivado-2022.1-standard
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
@@ -1,12 +1,3 @@
-default:
-  # Make all tasks interruptible by default
-  interruptible: true
-  retry:
-    max: 2
-    when:
-      - runner_system_failure
-      - stuck_or_timeout_failure
-
 include:
   - '/.ci/gitlab/common.yml'
   - '/.ci/gitlab/publish.yml'
@@ -39,10 +30,10 @@ tests:
     CI_PARENT_PIPELINE_SOURCE: $CI_PIPELINE_SOURCE
   parallel:
     matrix:
-      - GHC_VERSION: 9.6.2
+      - GHC_VERSION: 9.6.4
         RUN_TESTS: "always"
 
-      - GHC_VERSION: [9.4.6, 9.2.8]
+      - GHC_VERSION: [9.4.8, 9.2.8]
         RUN_TESTS: "nightly"
 
       - GHC_VERSION: 9.0.2