Skip to content

Automate functional test in the pipeline #2722

Automate functional test in the pipeline

Automate functional test in the pipeline #2722

Workflow file for this run

name: Conformance Testing
on:
push:
branches:
- main
- release-*
tags:
- "v[0-9]+.[0-9]+.[0-9]+*"
pull_request:
schedule:
- cron: "0 4 * * *" # run every day at 4am UTC
defaults:
run:
shell: bash
concurrency:
group: ${{ github.ref_name }}-conformance
cancel-in-progress: true
permissions:
contents: read
jobs:
conformance-tests:
name: Gateway Conformance Tests
runs-on: ubuntu-22.04
strategy:
matrix:
k8s-version: ["1.23.17", "latest"]
nginx-image: [nginx, nginx-plus]
enable-experimental: [true, false]
permissions:
contents: write # needed for uploading release artifacts
steps:
- name: Checkout Repository
uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
- name: Setup Golang Environment
uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491 # v5.0.0
with:
go-version: stable
- name: Set GOPATH
run: echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
- name: Docker Buildx
uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0
- name: NGF Docker meta
id: ngf-meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
with:
images: |
name=ghcr.io/nginxinc/nginx-gateway-fabric
tags: |
type=semver,pattern={{version}}
type=edge
type=ref,event=pr
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
- name: NGINX Docker meta
id: nginx-meta
uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5.5.1
with:
images: |
name=ghcr.io/nginxinc/nginx-gateway-fabric/${{ matrix.nginx-image }}
tags: |
type=semver,pattern={{version}}
type=edge
type=ref,event=pr
type=ref,event=branch,suffix=-rc,enable=${{ startsWith(github.ref, 'refs/heads/release') }}
- name: Prepare NGF files
run: |
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
ngf_tag=${{ steps.ngf-meta.outputs.version }}
make update-ngf-manifest${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
working-directory: ./conformance
- name: Build binary
uses: goreleaser/goreleaser-action@7ec5c2b0c6cdda6e8bbb49444bc797dd33d74dd8 # v5.0.0
with:
version: latest
args: build --snapshot --clean
- name: Build NGF Docker Image
uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
with:
file: build/Dockerfile
tags: ${{ steps.ngf-meta.outputs.tags }}
context: "."
target: goreleaser
load: true
cache-from: type=gha,scope=ngf
cache-to: type=gha,scope=ngf,mode=max
pull: true
- name: Build NGINX Docker Image
uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
with:
file: build/Dockerfile${{ matrix.nginx-image == 'nginx' && '.nginx' || '' }}${{ matrix.nginx-image == 'nginx-plus' && '.nginxplus' || ''}}
tags: ${{ steps.nginx-meta.outputs.tags }}
context: "."
load: true
cache-from: type=gha,scope=${{ matrix.nginx-image }}
cache-to: type=gha,scope=${{ matrix.nginx-image }},mode=max
pull: true
build-args: |
NJS_DIR=internal/mode/static/nginx/modules/src
NGINX_CONF_DIR=internal/mode/static/nginx/conf
BUILD_AGENT=gha
- name: Update Go Modules
if: ${{ github.event_name == 'schedule' }}
run: make update-go-modules
working-directory: ./conformance
- name: Build Test Docker Image
uses: docker/build-push-action@af5a7ed5ba88268d5278f7203fb52cd833f66d6e # v5.2.0
with:
file: conformance/tests/Dockerfile
tags: conformance-test-runner:${{ github.sha }}
context: "."
load: true
cache-from: type=gha
cache-to: type=gha,mode=max
pull: true
- name: Deploy Kubernetes
id: k8s
run: |
k8s_version=${{ matrix.k8s-version }}
make create-kind-cluster KIND_KUBE_CONFIG=${{ github.workspace }}/kube-${{ github.run_id }} ${{ ! contains(matrix.k8s-version, 'latest') && 'KIND_IMAGE=kindest/node:v${k8s_version}' || '' }}
echo "KUBECONFIG=${{ github.workspace }}/kube-${{ github.run_id }}" >> "$GITHUB_ENV"
working-directory: ./conformance
- name: Wait for release to exist
if: startsWith(github.ref, 'refs/tags/')
run: |
REF=${{ github.ref_name }}
until docker pull ghcr.io/nginxinc/nginx-gateway-fabric:${REF#v}; do sleep 5; done
until docker pull ghcr.io/nginxinc/nginx-gateway-fabric/nginx:${REF#v}; do sleep 5; done
- name: Setup conformance tests
run: |
ngf_prefix=ghcr.io/nginxinc/nginx-gateway-fabric
ngf_tag=${{ steps.ngf-meta.outputs.version }}
if [ ${{ github.event_name }} == "schedule" ]; then export GW_API_VERSION=main; fi
if [ ${{ startsWith(matrix.k8s-version, '1.23') || startsWith(matrix.k8s-version, '1.24') }} == "true" ]; then export INSTALL_WEBHOOK=true; fi
if [ ${{ matrix.enable-experimental }} == "true" ]; then export ENABLE_EXPERIMENTAL=true; fi
make install-ngf-local-no-build${{ matrix.nginx-image == 'nginx-plus' && '-with-plus' || ''}} PREFIX=${ngf_prefix} TAG=${ngf_tag}
working-directory: ./conformance
- name: Run conformance tests
run: |
make run-conformance-tests CONFORMANCE_TAG=${{ github.sha }} VERSION=${{ github.ref_name }}
core_result=$(cat conformance-profile.yaml | yq '.profiles[0].core.result')
extended_result=$(cat conformance-profile.yaml | yq '.profiles[0].extended.result')
if [ "${core_result}" == "failure" ] || [ "${extended_result}" == "failure" ]; then echo "Conformance test failed, see above for details." && exit 2; fi
working-directory: ./conformance
- name: Upload profile to release
if: ${{ matrix.k8s-version == 'latest' && startsWith(github.ref, 'refs/tags/') }}
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: gh release upload ${{ github.ref_name }} conformance-profile.yaml
working-directory: ./conformance