Merge pull request #85 from PermanentOrg/per-9558_deploy_archivematic…

…a_cleanup_cron Added Archivematica cleanup cron job to deployment
PermanentOrg · Jun 6, 2024 · d335a17 · d335a17
2 parents 4939f97 + 5df3eac
commit d335a17
Show file tree

Hide file tree

Showing 14 changed files with 466 additions and 55 deletions.
diff --git a/.github/workflows/dev_deploy.yml b/.github/workflows/dev_deploy.yml
@@ -9,29 +9,50 @@ jobs:
     uses: ./.github/workflows/test.yml
     secrets: inherit
 
-  build:
+  build_api:
     needs:
       - run_tests
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - name: Generate Image Tag
-        run: echo "IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        run: echo "API_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:api-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Build Image
-        run: docker build -t $IMAGE_TAG -f Dockerfile.api .
+        run: docker build -t $API_IMAGE_TAG -f Dockerfile.api .
       - name: AWS Login
         run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
-        run: docker push $IMAGE_TAG
+        run: docker push $API_IMAGE_TAG
+  build_am_cleanup:
+    needs:
+      - run_tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate Image Tag
+        run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Build Image
+        run: docker build -t $AM_CLEANUP_IMAGE_TAG -f Dockerfile.am_cleanup .
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $AM_CLEANUP_IMAGE_TAG
   deploy:
     needs:
-      - build
+      - build_api
+      - build_am_cleanup
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -42,8 +63,12 @@ jobs:
         uses: hashicorp/setup-terraform@v1
         with:
           cli_config_credentials_token: ${{ secrets.TERRAFORM_API_TOKEN }}
-      - name: Generate Image Tag
-        run: echo "IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+      - name: Generate API Image Tag
+        run: echo "API_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:api-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Generate Archivematica Cleanup Image Tag
+        run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Terraform Init
@@ -52,10 +77,15 @@ jobs:
       - name: Terraform Validate
         id: validate
         run: terraform validate -no-color
-      # Note that while Terraform requires us to pass a value of stela_staging_image below, it won't be used because the
-      # -target option restricts terraform to just updating the dev deployment
-      - name: Terraform Plan
-        id: plan
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$IMAGE_TAG" -var="stela_staging_image=$IMAGE_TAG" -target=kubernetes_deployment.stela_dev
-      - name: Terraform Apply
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$IMAGE_TAG" -var="stela_staging_image=$IMAGE_TAG" -target=kubernetes_deployment.stela_dev
+      # Note that while Terraform requires us to pass a values for the staging images below, they won't be used because
+      # the -target option restricts terraform to just updating the dev deployment
+      - name: Terraform Plan for API
+        id: plan_api
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
+      - name: Terraform Apply for API
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_deployment.stela_dev
+      - name: Terraform Plan for Archivematica cleanup
+        id: plan_am_cleanup
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
+      - name: Terraform Apply for Archivematica cleanup
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_cron_job_v1.archivematica_cleanup_dev
diff --git a/.github/workflows/full_test_deploy.yml b/.github/workflows/full_test_deploy.yml
@@ -6,29 +6,50 @@ jobs:
     uses: ./.github/workflows/test.yml
     secrets: inherit
 
-  build:
+  build_api:
     needs:
       - run_tests
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - name: Generate Image Tag
-        run: echo "IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        run: echo "API_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:api-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Build Image
-        run: docker build -t $IMAGE_TAG -f Dockerfile.api .
+        run: docker build -t $API_IMAGE_TAG -f Dockerfile.api .
       - name: AWS Login
         run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
-        run: docker push $IMAGE_TAG
+        run: docker push $API_IMAGE_TAG
+  build_am_cleanup:
+    needs:
+      - run_tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate Image Tag
+        run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Build Image
+        run: docker build -t $AM_CLEANUP_IMAGE_TAG -f Dockerfile.am_cleanup .
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $AM_CLEANUP_IMAGE_TAG
   deploy:
     needs:
-      - build
+      - build_api
+      - build_am_cleanup
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -39,8 +60,12 @@ jobs:
         uses: hashicorp/setup-terraform@v1
         with:
           cli_config_credentials_token: ${{ secrets.TERRAFORM_API_TOKEN }}
-      - name: Generate Image Tag
-        run: echo "IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+      - name: Generate API Image Tag
+        run: echo "API_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:api-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Generate Archivematica Cleanup Image Tag
+        run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Terraform Init
@@ -51,6 +76,6 @@ jobs:
         run: terraform validate -no-color
       - name: Terraform Plan
         id: plan
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$IMAGE_TAG" -var="stela_staging_image=$IMAGE_TAG"
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG"
       - name: Terraform Apply
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$IMAGE_TAG" -var="stela_staging_image=$IMAGE_TAG"
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG"
diff --git a/.github/workflows/prod_deploy.yml b/.github/workflows/prod_deploy.yml
@@ -8,29 +8,50 @@ jobs:
     uses: ./.github/workflows/test.yml
     secrets: inherit
 
-  build:
+  build_api:
     needs:
       - run_tests
     runs-on: ubuntu-20.04
     steps:
       - uses: actions/checkout@v3
       - name: Generate Image Tag
-        run: echo "IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        run: echo "API_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Build Image
-        run: docker build -t $IMAGE_TAG -f Dockerfile.api .
+        run: docker build -t $API_IMAGE_TAG -f Dockerfile.api .
       - name: AWS Login
         run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
         env:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Publish Image to ECR
-        run: docker push $IMAGE_TAG
+        run: docker push $API_IMAGE_TAG
+  build_am_cleanup:
+    needs:
+      - run_tests
+    runs-on: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v3
+      - name: Generate Image Tag
+        run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Build Image
+        run: docker build -t $AM_CLEANUP_IMAGE_TAG -f Dockerfile.am_cleanup .
+      - name: AWS Login
+        run: aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin 364159549467.dkr.ecr.$AWS_REGION.amazonaws.com
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Publish Image to ECR
+        run: docker push $AM_CLEANUP_IMAGE_TAG
   deploy_staging:
     needs:
-      - build
+      - build_api
+      - build_am_cleanup
     runs-on: ubuntu-20.04
     defaults:
       run:
@@ -41,8 +62,12 @@ jobs:
         uses: hashicorp/setup-terraform@v1
         with:
           cli_config_credentials_token: ${{ secrets.TERRAFORM_API_TOKEN }}
-      - name: Generate Image Tag
-        run: echo "IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+      - name: Generate API Image Tag
+        run: echo "API_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:api-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Generate Archivematica Cleanup Image Tag
+        run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Terraform Init
@@ -53,14 +78,20 @@ jobs:
         run: terraform validate -no-color
       # Note that while Terraform requires us to pass a value of stela_dev_image below, it won't be used because the
       # -target option restricts terraform to just updating the staging deployment
-      - name: Terraform Plan
-        id: plan
-        run: terraform plan -no-color -input=false -var="stela_dev_image=$IMAGE_TAG" -var="stela_staging_image=$IMAGE_TAG" -target=kubernetes_deployment.stela_staging
-      - name: Terraform Apply
-        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$IMAGE_TAG" -var="stela_staging_image=$IMAGE_TAG" -target=kubernetes_deployment.stela_staging
+      - name: Terraform Plan for API
+        id: plan_api
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_deployment.stela_staging
+      - name: Terraform Apply for API
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_deployment.stela_staging
+      - name: Terraform Plan for Archivematica cleanup
+        id: plan_am_cleanup
+        run: terraform plan -no-color -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_cron_job.archivematica_cleanup_staging
+      - name: Terraform Apply for Archivematica cleanup
+        run: terraform apply -auto-approve -input=false -var="stela_dev_image=$API_IMAGE_TAG" -var="stela_staging_image=$API_IMAGE_TAG" -var="archivematica_cleanup_dev_image=$AM_CLEANUP_IMAGE_TAG" -var="archivematica_cleanup_staging_image=$AM_CLEANUP_IMAGE_TAG" -target=kubernetes_cron_job.archivematica_cleanup_staging
   deploy_prod:
     needs:
-      - build
+      - build_api
+      - build_am_cleanup
       - deploy_staging
     runs-on: ubuntu-20.04
     environment: prod
@@ -73,8 +104,12 @@ jobs:
         uses: hashicorp/setup-terraform@v1
         with:
           cli_config_credentials_token: ${{ secrets.TERRAFORM_API_TOKEN }}
-      - name: Generate Image Tag
-        run: echo "IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+      - name: Generate API Image Tag
+        run: echo "API_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:api-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
+        env:
+          AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
+      - name: Generate Archivematica Cleanup Image Tag
+        run: echo "AM_CLEANUP_IMAGE_TAG=364159549467.dkr.ecr.$AWS_REGION.amazonaws.com/stela:am_cleanup-$([[ ${GITHUB_REF##*/} = main ]] && echo main || echo feature)-$(git rev-parse --short $GITHUB_SHA)" >> $GITHUB_ENV
         env:
           AWS_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
       - name: Terraform Init
@@ -85,6 +120,6 @@ jobs:
         run: terraform validate -no-color
       - name: Terraform Plan
         id: plan
-        run: terraform plan -no-color -input=false -var="stela_image=$IMAGE_TAG"
+        run: terraform plan -no-color -input=false -var="stela_image=$API_IMAGE_TAG" -var "archivematica_cleanup_image=$AM_CLEANUP_IMAGE_TAG"
       - name: Terraform Apply
-        run: terraform apply -auto-approve -input=false -var="stela_image=$IMAGE_TAG"
+        run: terraform apply -auto-approve -input=false -var="stela_image=$API_IMAGE_TAG" -var "archivematica_cleanup_image=$AM_CLEANUP_IMAGE_TAG"