Solve Duplicate Code in GitHub Action Workflows (#2589)

* Move LFS steps to separate action * Rename action.yml and use bash shell in each step * Test new environment setup * Specify Environment Name * Test cache environment key * Modify syntax * Modify update refdata too * Use new environment creation method in build docs and pre release workflow
tardis-sn · Apr 24, 2024 · 9b75d3d · 9b75d3d
1 parent 215ad61
commit 9b75d3d
Show file tree

Hide file tree

Showing 5 changed files with 159 additions and 206 deletions.
diff --git a/.github/actions/setup_lfs/action.yml b/.github/actions/setup_lfs/action.yml
@@ -0,0 +1,97 @@
+name: 'Setup LFS'
+description: 'Pull LFS repositories and caches them'
+
+inputs:
+  refdata-repo:
+    description: "tardis refdata repository"
+    required: false
+    default: 'tardis-sn/tardis-refdata'
+  regression-data-repo: 
+    description: "tardis regression data repository"
+    required: false
+    default: 'tardis-sn/tardis-regression-data'
+
+runs:
+    using: "composite"
+    steps:
+      - uses: actions/checkout@v4
+      - name: Clone Refdata Repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ inputs.refdata-repo }}
+          path: tardis-refdata
+          lfs: false
+
+      - name: Create LFS file list
+        run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id
+        working-directory: tardis-refdata
+        shell: bash
+
+      - name: Restore LFS cache
+        uses: actions/cache/restore@v3
+        id: lfs-cache-refdata
+        with:
+          path: tardis-refdata/.git/lfs
+          key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1
+
+      - name: Git LFS Pull
+        run: git lfs pull
+        working-directory: tardis-refdata
+        if: steps.lfs-cache-refdata.outputs.cache-hit != 'true'
+        shell: bash
+
+      - name: Git LFS Checkout
+        run: git lfs checkout
+        working-directory: tardis-refdata
+        if: steps.lfs-cache-refdata.outputs.cache-hit == 'true'
+        shell: bash
+
+      - name: Save LFS cache if not found
+        # uses fake ternary 
+        # for reference: https://github.com/orgs/community/discussions/26738#discussioncomment-3253176
+        if: ${{ steps.lfs-cache-refdata.outputs.cache-hit != 'true'  && always() || false }}
+        uses: actions/cache/save@v3
+        id: lfs-cache-refdata-save
+        with:
+          path: tardis-refdata/.git/lfs
+          key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1
+
+      - name: Clone tardis-sn/tardis-regression-data
+        uses: actions/checkout@v4
+        with:
+          repository:  ${{ inputs.regression-data-repo }}
+          path: tardis-regression-data
+
+      - name: Create LFS file list
+        run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id
+        working-directory: tardis-regression-data
+        shell: bash
+
+      - name: Restore LFS cache
+        uses: actions/cache/restore@v3
+        id: lfs-cache-regression-data
+        with:
+          path: tardis-regression-data/.git/lfs
+          key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-regression-data/.lfs-assets-id') }}-v1
+
+      - name: Git LFS Pull
+        run: git lfs pull
+        working-directory: tardis-regression-data
+        if: steps.lfs-cache-regression-data.outputs.cache-hit != 'true'
+        shell: bash
+
+      - name: Git LFS Checkout
+        run: git lfs checkout
+        working-directory: tardis-regression-data
+        if: steps.lfs-cache-regression-data.outputs.cache-hit == 'true'
+        shell: bash
+
+      - name: Save LFS cache if not found
+        # uses fake ternary 
+        # for reference: https://github.com/orgs/community/discussions/26738#discussioncomment-3253176
+        if: ${{ steps.lfs-cache-regression-data.outputs.cache-hit != 'true'  && always() || false }}
+        uses: actions/cache/save@v3
+        id: lfs-cache-regression-data-save
+        with:
+          path: tardis-regression-data/.git/lfs
+          key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-regression-data/.lfs-assets-id') }}-v1
diff --git a/.github/workflows/build-docs.yml b/.github/workflows/build-docs.yml
@@ -55,23 +55,20 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
         if: github.event_name == 'pull_request_target'
 
-      - name: Setup environment
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Generate Cache Key
+        run: | 
+          file_hash=$(cat conda-linux-64.lock | shasum -a 256 | cut -d' ' -f1)
+          echo "file_hash=$file_hash" >> "${GITHUB_OUTPUT}"
+        id: cache-environment-key
+
+      - uses: mamba-org/setup-micromamba@v1
         with:
-            miniforge-variant: Mambaforge
-            miniforge-version: latest
-            activate-environment: tardis
-            use-mamba: true
-
-      - uses: actions/cache@v2
-        with:
-          path: /usr/share/miniconda3/envs/tardis
-          key: conda-linux-64-${{ hashFiles('conda-linux-64.lock') }}-${{ env.CACHE_NUMBER }}
-        id: cache-conda
-
-      - name: Update environment
-        run: mamba update -n tardis --file conda-linux-64.lock
-        if: steps.cache-conda.outputs.cache-hit != 'true'
+          environment-file: conda-linux-64.lock
+          cache-environment-key: ${{ steps.cache-environment-key.outputs.file_hash }}
+          cache-downloads-key: ${{ steps.cache-environment-key.outputs.file_hash }}
+          environment-name: tardis
+          cache-environment: true
+          cache-downloads: true
 
       - name: Install package
         run: pip install -e .

diff --git a/.github/workflows/pre-release.yml b/.github/workflows/pre-release.yml
@@ -30,14 +30,20 @@ jobs:
       - name: Download Lock File
         run:  wget -q https://raw.githubusercontent.com/tardis-sn/tardis/master/conda-linux-64.lock
 
-      - name: Setup environment
-        uses: conda-incubator/setup-miniconda@v2
+      - name: Generate Cache Key
+        run: | 
+          file_hash=$(cat conda-linux-64.lock | shasum -a 256 | cut -d' ' -f1)
+          echo "file_hash=$file_hash" >> "${GITHUB_OUTPUT}"
+        id: cache-environment-key
+
+      - uses: mamba-org/setup-micromamba@v1
         with:
-            miniforge-variant: Mambaforge
-            miniforge-version: latest
-            environment-file: conda-linux-64.lock
-            activate-environment: tardis_zenodo
-            use-mamba: true
+          environment-file: conda-linux-64.lock
+          cache-environment-key: ${{ steps.cache-environment-key.outputs.file_hash }}
+          cache-downloads-key: ${{ steps.cache-environment-key.outputs.file_hash }}
+          environment-name: tardis
+          cache-environment: true
+          cache-downloads: true
 
       - name: Run Notebook
         run: jupyter nbconvert gather_data.ipynb --to html --execute --ExecutePreprocessor.timeout=6000

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
@@ -46,108 +46,32 @@ jobs:
     name: ${{ matrix.label }}-pip-${{ matrix.pip }}
     runs-on: ${{ matrix.os }}
     steps:
-      - uses: actions/checkout@v2
-        if: matrix.pip == false
-
-      - name: Clone tardis-sn/tardis-refdata
-        uses: actions/checkout@v2
-        with:
-          repository: tardis-sn/tardis-refdata
-          path: tardis-refdata
-          lfs: false
+      - uses: actions/checkout@v4
 
-      - name: Create LFS file list
-        run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id
-        working-directory: tardis-refdata
-
-      - name: Restore LFS cache
-        uses: actions/cache/restore@v3
-        id: lfs-cache-refdata
-        with:
-          path: tardis-refdata/.git/lfs
-          key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1
-
-      - name: Git LFS Pull
-        run: git lfs pull
-        working-directory: tardis-refdata
-        if: steps.lfs-cache-refdata.outputs.cache-hit != 'true'
-
-      - name: Git LFS Checkout
-        run: git lfs checkout
-        working-directory: tardis-refdata
-        if: steps.lfs-cache-refdata.outputs.cache-hit == 'true'
-
-      - name: Save LFS cache if not found
-        # uses fake ternary 
-        # for reference: https://github.com/orgs/community/discussions/26738#discussioncomment-3253176
-        if: ${{ steps.lfs-cache-refdata.outputs.cache-hit != 'true'  && always() || false }}
-        uses: actions/cache/save@v3
-        id: lfs-cache-refdata-save
-        with:
-          path: tardis-refdata/.git/lfs
-          key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-refdata/.lfs-assets-id') }}-v1
-
-      - name: Clone tardis-sn/tardis-regression-data
-        uses: actions/checkout@v4
-        with:
-          repository: tardis-sn/tardis-regression-data
-          path: tardis-regression-data
-
-      - name: Create LFS file list
-        run: git lfs ls-files -l | cut -d' ' -f1 | sort > .lfs-assets-id
-        working-directory: tardis-regression-data
-
-      - name: Restore LFS cache
-        uses: actions/cache/restore@v3
-        id: lfs-cache-regression-data
-        with:
-          path: tardis-regression-data/.git/lfs
-          key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-regression-data/.lfs-assets-id') }}-v1
-
-      - name: Git LFS Pull
-        run: git lfs pull
-        working-directory: tardis-regression-data
-        if: steps.lfs-cache-regression-data.outputs.cache-hit != 'true'
-
-      - name: Git LFS Checkout
-        run: git lfs checkout
-        working-directory: tardis-regression-data
-        if: steps.lfs-cache-regression-data.outputs.cache-hit == 'true'
-
-      - name: Save LFS cache if not found
-        # uses fake ternary 
-        # for reference: https://github.com/orgs/community/discussions/26738#discussioncomment-3253176
-        if: ${{ steps.lfs-cache-regression-data.outputs.cache-hit != 'true'  && always() || false }}
-        uses: actions/cache/save@v3
-        id: lfs-cache-regression-data-save
-        with:
-          path: tardis-regression-data/.git/lfs
-          key: ${{ runner.os }}-lfs-${{ hashFiles('tardis-regression-data/.lfs-assets-id') }}-v1
-
-      - name: Setup environment
-        uses: conda-incubator/setup-miniconda@v2
-        with:
-            miniforge-variant: Mambaforge
-            miniforge-version: latest
-            activate-environment: tardis
-            use-mamba: true
+      - name: Setup LFS
+        uses: ./.github/actions/setup_lfs
 
       - name: Download Lock File
         run:  wget -q https://raw.githubusercontent.com/tardis-sn/tardis/master/conda-${{ matrix.label }}.lock
         if: matrix.pip == true
-
-      - uses: actions/cache@v2
+
+      - name: Generate Cache Key
+        run: | 
+          file_hash=$(cat conda-${{ matrix.label }}.lock | shasum -a 256 | cut -d' ' -f1)
+          echo "file_hash=$file_hash" >> "${GITHUB_OUTPUT}"
+        id: cache-environment-key
+
+      - uses: mamba-org/setup-micromamba@v1
         with:
-          path: ${{ matrix.prefix }}
-          key: conda-${{ matrix.label }}-${{ hashFiles('conda-${{ matrix.label }}.lock') }}-${{ env.CACHE_NUMBER }}
-        id: cache-conda
+          environment-file: conda-${{ matrix.label }}.lock
+          cache-environment-key: ${{ steps.cache-environment-key.outputs.file_hash }}
+          cache-downloads-key: ${{ steps.cache-environment-key.outputs.file_hash }}
+          environment-name: tardis
+          cache-environment: true
+          cache-downloads: true
 
-      - name: Update environment
-        run: mamba update -n tardis --file conda-${{ matrix.label }}.lock
-        if: steps.cache-conda.outputs.cache-hit != 'true'
-
       - name: Install package editable
-        run: | 
+        run: |
           pip install -e .
           echo "TARDIS_PIP_PATH=tardis" >> $GITHUB_ENV
         if: matrix.pip == false