GenSpectrum · Taepper · Oct 24, 2024 · Oct 24, 2024 · Nov 14, 2024 · Nov 14, 2024
diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.yml
@@ -0,0 +1,90 @@
+name: "Dependency Image"
+
+on:
+  push:
+  workflow_dispatch:
+
+env:
+  DOCKER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-dependencies
+
+jobs:
+  dependencyImage:
+    name: Build Docker Image Dependencies
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform:
+          ['linux/amd64', 'linux/arm64']
+    permissions:
+      packages: write
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Generate dependency files hash
+        id: files-hash
+        run: |
+          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
+          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+
+      - name: Docker metadata
+        id: dockerMetadata
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
+            type=raw,value=${{ env.DIR_HASH }}
+            type=sha,format=long,prefix=commit-
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Check if image exists
+        run: |
+          if [[ $(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1) ]]; then
+            echo "Cache hit, do not require rebuild of this docker image. Retagging old image instead."
+            echo "CACHE_HIT=true" >> $GITHUB_ENV
+          else
+            echo "CACHE_HIT=false" >> $GITHUB_ENV
+          fi
+
+      - name: Determine platform build necessity
+        if: env.CACHE_HIT == 'false'
+        id: platform-check
+        run: |
+          if [[ "${{ matrix.platform }}" == "linux/amd64" || "${{ github.event_name }}" == "workflow_dispatch" || "${{ github.ref }}" == "refs/heads/main" ]]; then
+            echo "PLATFORM_BUILD=true" >> $GITHUB_ENV
+          else
+            echo "Skipping build for this platform"
+            echo "PLATFORM_BUILD=false" >> $GITHUB_ENV
+          fi
+        shell: bash
+
+      - name: Set up Docker Buildx
+        if: env.CACHE_HIT == 'false' && env.PLATFORM_BUILD == 'true'
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push image if input files changed
+        if: env.CACHE_HIT == 'false' && env.PLATFORM_BUILD == 'true'
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: Dockerfile_dependencies
+          push: true
+          tags: ${{ steps.dockerMetadata.outputs.tags }}
+          cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
+          cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
+          platforms: ${{ matrix.platform }}
+
+      - name: Retag and push existing image if cache hit
+        if: env.CACHE_HIT == 'true' && env.PLATFORM_BUILD == 'true'
+        run: |
+          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
+          for TAG in "${TAGS[@]}"; do
+            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }}
+          done
diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml
@@ -0,0 +1,22 @@
+name: Formatting
+on:
+  push:
+
+jobs:
+  formatting-check:
+    name: Formatting Check
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        path:
+          - check: 'src'
+          - check: 'include'
+            exclude: '(PerfEvent.hpp)'
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run clang-format style check
+        uses: jidicula/clang-format-action@v4.13.0
+        with:
+          clang-format-version: '17'
+          check-path: ${{ matrix.path['check'] }}
+          exclude-regex: ${{ matrix.path['exclude'] }}
diff --git a/.github/workflows/ci.yml → .github/workflows/imageAndTests.yml b/.github/workflows/ci.yml → .github/workflows/imageAndTests.yml
@@ -2,6 +2,7 @@ name: LAPIS-SILO
 
 on:
   push:
+  workflow_dispatch:
 
 concurrency:
   group: ci-${{ github.ref }}
@@ -12,101 +13,38 @@ env:
   DOCKER_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo
 
 jobs:
-  formatting-check:
-    name: Formatting Check
+  waitForDependencyImage:
+    name: Wait for dependency image
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        path:
-          - check: 'src'
-          - check: 'include'
-            exclude: '(PerfEvent.hpp)'
     steps:
-      - uses: actions/checkout@v4
-      - name: Run clang-format style check
-        uses: jidicula/clang-format-action@v4.13.0
-        with:
-          clang-format-version: '17'
-          check-path: ${{ matrix.path['check'] }}
-          exclude-regex: ${{ matrix.path['exclude'] }}
+      - uses: actions/checkout@v2
 
-  dependencyImage:
-    name: Build Docker Image Dependencies
-    runs-on: ubuntu-latest
-    permissions:
-      packages: write
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Generate dependency files hash
-        id: files-hash
+      - shell: bash
+        name: Print job function
         run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
-          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+          echo "Calling waiter for workflow 'Dependency Image', sha: ${{ github.sha }}"
 
-      - name: Docker metadata
-        id: dockerMetadata
-        uses: docker/metadata-action@v5
+      - name: Wait
+        uses: lewagon/wait-on-check-action@v1.3.4
         with:
-          images: ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}
-          tags: |
-            type=ref,event=branch
-            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
-            type=raw,value=${{ env.DIR_HASH }}
-            type=sha,format=long,prefix=commit-
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Check if image exists
-        id: check-image
-        run: |
-          EXISTS=$(docker manifest inspect ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
-          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        if: env.CACHE_HIT == 'false'
-        uses: docker/setup-buildx-action@v3
-
-      - name: Build and push image if input files changed
-        if: env.CACHE_HIT == 'false'
-        uses: docker/build-push-action@v6
-        with:
-          context: .
-          file: Dockerfile_dependencies
-          push: true
-          tags: ${{ steps.dockerMetadata.outputs.tags }}
-          cache-from: type=gha,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
-          cache-to: type=gha,mode=min,ref=builder-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile') }}
-          platforms: linux/amd64,linux/arm64
-
-      - name: Retag and push existing image if cache hit
-        if: env.CACHE_HIT == 'true'
-        run: |
-          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
-          for TAG in "${TAGS[@]}"; do
-            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }}
-          done
+          ref: ${{ github.sha }}
+          check-name: "Dependency Image"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 3
 
   dockerImageUnitTests:
     name: Build Docker Image and Run Unit Tests
     runs-on: ubuntu-latest
-    needs: dependencyImage
+    needs: waitForDependencyImage
+    strategy:
+      matrix:
+        args:
+          ['linux/amd64', 'linux/arm64']
     permissions:
       packages: write
     steps:
       - uses: actions/checkout@v4
 
-      - name: Generate dependency files hash
-        id: files-hash
-        run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/ci.yml', './Dockerfile_dependencies') }})
-          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
-
       - name: Docker metadata
         id: dockerMetadata
         uses: docker/metadata-action@v5
@@ -137,7 +75,7 @@ jobs:
           cache-from: type=gha,ref=${{ github.ref_name }}-image-cache
           cache-to: type=gha,mode=min,ref=${{ github.ref_name }}-image-cache
           build-args: |
-            DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:${{ env.DIR_HASH }}
+            DEPENDENCY_IMAGE=${{ env.DOCKER_DEPENDENCY_IMAGE_NAME }}:commit-${{ github.sha }}
 
       - name: Run tests
         uses: addnab/docker-run-action@v3
@@ -176,7 +114,7 @@ jobs:
       - uses: actions/cache@v4
         with:
           path: ~/.npm
-          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
+          key: "${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}"
 
       - name: npm install
         run: cd endToEndTests && npm ci

diff --git a/.github/workflows/linter.yml b/.github/workflows/linter.yml
@@ -7,79 +7,34 @@ concurrency:
   group: linter-${{ github.ref }}
   cancel-in-progress: true
 
-env:
-  DOCKER_LINTER_DEPENDENCY_IMAGE_NAME: ghcr.io/genspectrum/lapis-silo-linter-dependencies
-
 jobs:
-  linterDependencies:
-    name: Build linter dependencies
+  waitForDependencyImage:
+    name: Wait for dependency image
     runs-on: ubuntu-latest
-    permissions:
-      packages: write
     steps:
-      - uses: actions/checkout@v4
-
-      - name: Generate dependency files hash
-        id: files-hash
-        run: |
-          DIR_HASH=$(echo -n ${{ hashFiles('conanfile.py', 'conanprofile.docker', '.github/workflows/linter.yml', './Dockerfile_linter_dependencies') }})
-          echo "DIR_HASH=$DIR_HASH" >> $GITHUB_ENV
+      - uses: actions/checkout@v2
 
-      - name: Docker metadata
-        id: dockerMetadata
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}
-          tags: |
-            type=ref,event=branch
-            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
-            type=raw,value=${{ env.DIR_HASH }}
-            type=sha,format=long,prefix=commit-
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Check if image exists
-        id: check-image
+      - shell: bash
+        name: Print job function
         run: |
-          EXISTS=$(docker manifest inspect ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }} > /dev/null 2>&1 && echo "true" || echo "false")
-          echo "CACHE_HIT=$EXISTS" >> $GITHUB_ENV
-
-      - name: Set up Docker Buildx
-        if: env.CACHE_HIT == 'false'
-        uses: docker/setup-buildx-action@v3
+          echo "Calling waiter for workflow 'Dependency Image', sha: ${{ github.sha }}"
 
-      - name: Build linter dependencies image
-        if: env.CACHE_HIT == 'false'
-        uses: docker/build-push-action@v6
+      - name: Wait
+        uses: lewagon/wait-on-check-action@v1.3.4
         with:
-          context: .
-          push: true
-          tags: ${{ steps.dockerMetadata.outputs.tags }}
-          file: ./Dockerfile_linter_dependencies
-          cache-from: type=gha,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }}
-          cache-to: type=gha,mode=min,ref=linter-dependencies-image-cache-${{ hashFiles('conanfile.py', 'Dockerfile_linter') }}
-
-      - name: Retag and push existing image if cache hit
-        if: env.CACHE_HIT == 'true'
-        run: |
-          TAGS=(${{ steps.dockerMetadata.outputs.tags }})
-          for TAG in "${TAGS[@]}"; do
-            docker buildx imagetools create --tag $TAG ${{ env.DOCKER_LINTER_DEPENDENCY_IMAGE_NAME }}:${{ env.DIR_HASH }}
-          done
+          ref: ${{ github.sha }}
+          check-name: "Dependency Image"
+          repo-token: ${{ secrets.GITHUB_TOKEN }}
+          wait-interval: 3
 
   linter:
     name: Build And Run linter
-    needs: linterDependencies
+    needs: waitForDependencyImage
     runs-on: ubuntu-latest
     env:
       PR_NUMBER: ${{ github.event.number }}
     container:
-      image: ghcr.io/genspectrum/lapis-silo-linter-dependencies:commit-${{ github.sha }}
+      image: ghcr.io/genspectrum/lapis-silo-dependencies:commit-${{ github.sha }}"
     steps:
       - uses: actions/checkout@v4
 

diff --git a/Dockerfile_dependencies b/Dockerfile_dependencies
@@ -8,6 +8,11 @@ RUN apt update && apt dist-upgrade -y \
 
 RUN pip install conan==2.8.1
 
+RUN apt install -y software-properties-common wget gnupg lsb-release wget jq curl \
+    && wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc \
+    && add-apt-repository 'deb http://apt.llvm.org/jammy/  llvm-toolchain-jammy main' \
+    && apt install -y clang-tidy-15
+
 WORKDIR /src
 COPY conanfile.py conanprofile.docker conanprofile.docker_arm ./
 RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
@@ -17,3 +22,4 @@ RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then \
     fi
 
 RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build/Release/generators
+RUN conan install . --build=missing --profile ./conanprofile --profile:build ./conanprofile --output-folder=build/Debug/generators
diff --git a/Dockerfile_linter b/Dockerfile_linter
@@ -1,4 +1,4 @@
-ARG DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-linter-dependencies:latest
+ARG DEPENDENCY_IMAGE=ghcr.io/genspectrum/lapis-silo-dependencies:latest
 
 FROM $DEPENDENCY_IMAGE AS linter
 ARG THREADS=4
@@ -7,4 +7,5 @@ WORKDIR /src
 
 COPY . ./
 
-RUN python3 ./build_with_conan.py --parallel $THREADS --build_with_clang_tidy
+RUN cmake -D BUILD_WITH_CLANG_TIDY=ON -D CMAKE_BUILD_TYPE=Debug -B build/Debug
+RUN cmake --build build/Debug