azure-e2e-test #245
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: azure-e2e-test | |
| permissions: | |
| id-token: write | |
| contents: read | |
| env: | |
| CLUSTER_NAME: "e2e-test-${{ github.run_id }}-${{ github.run_attempt }}" | |
| TEST_PROVISION_FILE: "${{ github.workspace }}/provision_azure${{ github.run_id }}.properties" | |
| CLOUD_PROVIDER: "azure" | |
| BUILTIN_CLOUD_PROVIDERS: "azure" | |
| TEST_E2E_CREATE_RG: "no" | |
| ACR_URL: "${{ vars.AZURE_ACR_URL }}" | |
| on: | |
| schedule: | |
| # Runs "at midnight every day" (see https://crontab.guru) | |
| # will base on default branch `main` | |
| - cron: '0 0 * * *' | |
| workflow_dispatch: | |
| inputs: | |
| podvm-image-id: | |
| type: string | |
| description: prebuilt podvm image | |
| caa-image: | |
| type: string | |
| description: prebuilt caa image | |
| kbs-image-id: | |
| type: string | |
| description: prebuild kbs image | |
| kbs-image-tag: | |
| type: string | |
| description: prebuild kbs image tag | |
| jobs: | |
| generate-podvm-image-version: | |
| if: github.event.inputs.podvm-image-id == '' | |
| runs-on: ubuntu-latest | |
| outputs: | |
| image-version: "${{ steps.generate-image-version.outputs.image-version }}" | |
| steps: | |
| - name: Generate PodVM image version | |
| id: generate-image-version | |
| run: | | |
| unique_version="$(date '+%m.%d.%H%M%S')${{ github.run_attempt }}" | |
| echo "Generated unique version for the image as: ${unique_version}" | |
| echo "image-version=${unique_version}" >> "$GITHUB_OUTPUT" | |
| build-podvm-image: | |
| uses: ./.github/workflows/azure-podvm-image-build.yml | |
| needs: | |
| - generate-podvm-image-version | |
| if: github.event.inputs.podvm-image-id == '' | |
| secrets: inherit | |
| with: | |
| image-version: ${{ needs.generate-podvm-image-version.outputs.image-version }} | |
| build-caa-container-image: | |
| if: github.event.inputs.caa-image == '' | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: src/cloud-api-adaptor | |
| outputs: | |
| caa-image: "${{ steps.build-container.outputs.caa-image }}" | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Extract go version number | |
| run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV" | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| cache-dependency-path: go.sum | |
| - uses: azure/login@v1 | |
| name: 'Az CLI login' | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Install build dependencies | |
| run: | | |
| sudo apt-get update -y | |
| sudo apt-get install -y libvirt-dev | |
| - name: Build container image | |
| id: build-container | |
| env: | |
| ARCHES: "linux/amd64" | |
| RELEASE_BUILD: "false" | |
| run: | | |
| az acr login --name "$ACR_URL" | |
| # This builds image in the following format: | |
| # $ACR_URL/cloud-api-adaptor:dev-COMMIT | |
| make image "registry=${ACR_URL}" | |
| echo "caa-image=${ACR_URL}/cloud-api-adaptor:dev-${GITHUB_SHA}" >> "$GITHUB_OUTPUT" | |
| install-aks: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: src/cloud-api-adaptor | |
| needs: | |
| - generate-podvm-image-version | |
| - build-caa-container-image | |
| if: always() && !cancelled() && needs.build-caa-container-image.result != 'failure' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Extract go version number | |
| run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV" | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Extract provisioner environment | |
| run: | | |
| echo "KBS_IMAGE=$(yq -e '.oci.kbs.registry' versions.yaml)" >> "$GITHUB_ENV" | |
| echo "KBS_IMAGE_TAG=$(yq -e '.oci.kbs.tag' versions.yaml)" >> "$GITHUB_ENV" | |
| - name: Create provisioner file | |
| env: | |
| AZURE_IMAGE_ID: ${{ github.event.inputs.podvm-image-id || format('/CommunityGalleries/{0}/images/{1}/Versions/{2}', vars.AZURE_COMMUNITY_GALLERY_NAME, vars.AZURE_PODVM_IMAGE_DEF_NAME, needs.generate-podvm-image-version.outputs.image-version) }} | |
| CAA_IMAGE: "${{ github.event.inputs.caa-image || needs.build-caa-container-image.outputs.caa-image }}" | |
| KBS_IMAGE: ${{ github.event.inputs.kbs-image-id || env.KBS_IMAGE }} | |
| KBS_IMAGE_TAG: ${{ github.event.inputs.kbs-image-tag || env.KBS_IMAGE_TAG}} | |
| run: | | |
| cat << EOF > "$TEST_PROVISION_FILE" | |
| AZURE_SUBSCRIPTION_ID="${{ secrets.AZURE_SUBSCRIPTION_ID }}" | |
| AZURE_CLIENT_ID="${{ secrets.AZURE_CLIENT_ID }}" | |
| RESOURCE_GROUP_NAME="${{ secrets.AZURE_RESOURCE_GROUP }}" | |
| CLUSTER_NAME="$CLUSTER_NAME" | |
| LOCATION="${{ secrets.AZURE_REGION }}" | |
| SSH_KEY_ID="id_rsa.pub" | |
| AZURE_IMAGE_ID="$AZURE_IMAGE_ID" | |
| IS_CI_MANAGED_CLUSTER="true" | |
| MANAGED_IDENTITY_NAME="${{ secrets.AZURE_MANAGED_IDENTITY_NAME}}" | |
| CAA_IMAGE="${CAA_IMAGE}" | |
| KBS_IMAGE="${KBS_IMAGE}" | |
| KBS_IMAGE_TAG="${KBS_IMAGE_TAG}" | |
| EOF | |
| cat "$TEST_PROVISION_FILE" | |
| # assert that no variable is unset | |
| ! grep -E '=x$|=""$' "$TEST_PROVISION_FILE" | |
| - name: Create public ssh key | |
| run: ssh-keygen -t rsa -b 4096 -f install/overlays/azure/id_rsa -N "" -C dev@coco.io | |
| - name: Save the configuration created here | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: | | |
| src/cloud-api-adaptor/install/overlays/azure/id_rsa.pub | |
| ${{ env.TEST_PROVISION_FILE }} | |
| name: e2e-configuration | |
| - uses: azure/login@v1 | |
| name: 'Az CLI login' | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| # Install AKS cluster in parallel with the podvm image build. | |
| - name: Run provisioner | |
| working-directory: src/cloud-api-adaptor/test/tools | |
| run: | | |
| make caa-provisioner-cli | |
| ./caa-provisioner-cli -action=createcluster | |
| run-e2e-test: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: src/cloud-api-adaptor | |
| needs: | |
| - build-podvm-image | |
| - build-caa-container-image | |
| - install-aks | |
| - generate-podvm-image-version | |
| if: always() && !cancelled() && needs.build-podvm-image.result != 'failure' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Extract go version number | |
| run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV" | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - uses: azure/login@v1 | |
| name: 'Az CLI login' | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| - name: Restore the configuration created before | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: e2e-configuration | |
| - name: Add AKS Cluster Subnet ID to test provision file | |
| run: | | |
| NODE_RESOURCE_GROUP="$(az aks show -g ${{ secrets.AZURE_RESOURCE_GROUP }} -n "$CLUSTER_NAME" --query nodeResourceGroup -o tsv)" | |
| SUBNET_ID="$(az network vnet list -g "$NODE_RESOURCE_GROUP" --query '[0].subnets[0].id' -o tsv)" | |
| test -n "$SUBNET_ID" | |
| echo "AZURE_SUBNET_ID=\"${SUBNET_ID}\"" >> "$TEST_PROVISION_FILE" | |
| - name: Extract kbs reference | |
| run: echo "KBS_VERSION=$(yq -e '.git.kbs.reference' versions.yaml)" >> "$GITHUB_ENV" | |
| - name: Checkout kbs Repository | |
| run: | | |
| git clone https://github.com/confidential-containers/trustee test/trustee | |
| pushd test/trustee | |
| git checkout "${KBS_VERSION}" | |
| popd | |
| - name: Run e2e test | |
| env: | |
| TEST_PROVISION: "no" | |
| DEPLOY_KBS: "yes" | |
| run: | | |
| # Since we install the cluster in parallel, we need to get the credentials here. | |
| az aks get-credentials \ | |
| --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ | |
| --name "${CLUSTER_NAME}" | |
| make test-e2e | |
| cleanup-resources: | |
| runs-on: ubuntu-latest | |
| defaults: | |
| run: | |
| working-directory: src/cloud-api-adaptor | |
| needs: | |
| - generate-podvm-image-version | |
| - build-podvm-image | |
| - build-caa-container-image | |
| - run-e2e-test | |
| if: always() | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Extract go version number | |
| run: echo "GO_VERSION=$(yq -e '.tools.golang' versions.yaml)" >> "$GITHUB_ENV" | |
| - name: Set up Go environment | |
| uses: actions/setup-go@v4 | |
| with: | |
| go-version: ${{ env.GO_VERSION }} | |
| - name: Restore the configuration created before | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: e2e-configuration | |
| - uses: azure/login@v1 | |
| name: 'Az CLI login' | |
| with: | |
| client-id: ${{ secrets.AZURE_CLIENT_ID }} | |
| subscription-id: ${{ secrets.AZURE_SUBSCRIPTION_ID }} | |
| tenant-id: ${{ secrets.AZURE_TENANT_ID }} | |
| # Clean up step, run regardless of the failure state. | |
| - name: Run deprovisioner | |
| working-directory: src/cloud-api-adaptor/test/tools | |
| run: | | |
| make caa-provisioner-cli | |
| # Ignore the error if the deprovision fails. | |
| ./caa-provisioner-cli -action=deprovision || true | |
| - name: Remove podvm image | |
| if: github.event.inputs.podvm-image-id == '' | |
| run: | | |
| # Delete the Pod VM image from the gallery. | |
| az sig image-version delete \ | |
| --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ | |
| --gallery-name ${{ secrets.AZURE_PODVM_GALLERY_NAME }} \ | |
| --gallery-image-definition ${{ vars.AZURE_PODVM_IMAGE_DEF_NAME }} \ | |
| --gallery-image-version "${{ needs.generate-podvm-image-version.outputs.image-version }}" || true | |
| - name: Remove container image | |
| if: github.event.inputs.caa-image == '' | |
| run: | | |
| # Delete the CAA container image built for this run. | |
| suffix=".azurecr.io" | |
| registry_name="${ACR_URL%"${suffix}"}" | |
| az acr repository delete \ | |
| --name "${registry_name}" \ | |
| --image "${ACR_URL}/cloud-api-adaptor:dev-${GITHUB_SHA}" \ | |
| --yes || true | |
| - name: Remove AKS cluster | |
| run: | | |
| # Delete the cluster even if it has been deleted already or does not exists. | |
| az aks delete \ | |
| --name "${CLUSTER_NAME}" \ | |
| --resource-group ${{ secrets.AZURE_RESOURCE_GROUP }} \ | |
| --no-wait \ | |
| --yes || true |