E2e tests fixes after 0.19 updates #150
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| push: | |
| branches: | |
| - master | |
| - develop | |
| paths-ignore: | |
| - "docs/**" | |
| pull_request: | |
| branches: | |
| - master | |
| - develop | |
| paths-ignore: | |
| - "docs/**" | |
| jobs: | |
| unit_tests: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| matrix: | |
| python-version: [ '3.8', '3.9', '3.10', '3.11'] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup python ${{ matrix.python-version }} | |
| uses: actions/setup-python@v2.2.1 | |
| with: | |
| python-version: ${{ matrix.python-version }} | |
| - name: Setup poetry | |
| run: | | |
| python -V | |
| python -m pip install poetry | |
| - name: Check pre-commit status | |
| run: | | |
| poetry install -v | |
| poetry run pre-commit run --all-files | |
| - name: Test with tox | |
| run: | | |
| pip install tox-pip-version tox-gh-actions "tox<4.0.0" | |
| tox -v | |
| - name: Store coverage reports | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: coverage-${{ matrix.python-version }} | |
| path: coverage.xml | |
| if-no-files-found: error | |
| sonarcloud: | |
| runs-on: ubuntu-latest | |
| needs: unit_tests | |
| steps: | |
| - uses: actions/checkout@v2 | |
| with: | |
| fetch-depth: 0 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: coverage-3.9 | |
| path: . | |
| - name: SonarCloud Scan | |
| uses: sonarsource/sonarcloud-github-action@master | |
| with: | |
| projectBaseDir: ${{ github.workspace }} | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SONAR_TOKEN: ${{ secrets.SONARCLOUD_TOKEN }} | |
| codeql: | |
| runs-on: ubuntu-latest | |
| needs: unit_tests | |
| permissions: | |
| actions: read | |
| contents: read | |
| security-events: write | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v3 | |
| # Initializes the CodeQL tools for scanning. | |
| - name: Initialize CodeQL | |
| uses: github/codeql-action/init@v2 | |
| with: | |
| languages: python | |
| - name: Autobuild | |
| uses: github/codeql-action/autobuild@v2 | |
| - name: Perform CodeQL Analysis | |
| uses: github/codeql-action/analyze@v2 | |
| e2e_tests: | |
| if: (github.event.pull_request.head.repo.full_name || github.push.repository.full_name) == github.repository | |
| runs-on: ubuntu-latest | |
| needs: [unit_tests, sonarcloud, codeql] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| - name: Setup python | |
| uses: actions/setup-python@v2.2.1 | |
| with: | |
| python-version: '3.10' | |
| - name: Setup virtualenv | |
| run: | | |
| python -V | |
| python -m pip install virtualenv | |
| virtualenv venv | |
| source venv/bin/activate | |
| - name: Initialize kedro spaceflights project | |
| run: | | |
| pip install . | |
| kedro new --starter spaceflights-pandas --config tests/e2e/starter-config.yml --verbose | |
| - name: Install project dependencies | |
| working-directory: ./spaceflights | |
| run: | | |
| echo "kedro-kubeflow[gcp] @ git+https://github.com/getindata/kedro-kubeflow.git@$GITHUB_SHA" >> src/requirements.txt | |
| echo "kedro-docker" >> src/requirements.txt | |
| sed -i '/kedro-telemetry/d' src/requirements.txt | |
| cat src/requirements.txt | |
| pip install -r src/requirements.txt | |
| - name: Init and update configuration | |
| working-directory: ./spaceflights | |
| run: | | |
| sed -i "s/\(CONFIG_LOADER_ARGS.*\)$/from omegaconf.resolvers import oc\n\1\n \"custom_resolvers\": { \"oc.env\": oc.env },/g" src/spaceflights/settings.py | |
| kedro docker init | |
| kedro kubeflow init http://localhost:9000 | |
| echo "!data/01_raw" >> .dockerignore | |
| cp ../tests/e2e/catalog.yml conf/base/catalog.yml | |
| rm -f conf/base/kubeflow.y* | |
| cp ../tests/e2e/kubeflow.yml conf/base/kubeflow.yml | |
| - name: Prepare docker env | |
| uses: docker/setup-buildx-action@v1 | |
| with: | |
| install: true | |
| - name: Build pipeline docker image | |
| working-directory: ./spaceflights | |
| run: | | |
| docker build --build-arg BASE_IMAGE=python:3.8-buster --tag kedro-kubeflow-e2e:latest --load . | |
| - name: Publish docker image to GCR | |
| uses: mattes/gce-docker-push-action@v1 | |
| with: | |
| creds: ${{ secrets.GOOGLE_CLOUD_TOKEN_JSON }} | |
| src: kedro-kubeflow-e2e:latest | |
| dst: gcr.io/gid-ml-ops-sandbox/kedro-kubeflow-e2e:${{ github.sha }} | |
| - name: Set up GCP Credentials | |
| uses: google-github-actions/auth@v0.6.0 | |
| with: | |
| credentials_json: ${{ secrets.GOOGLE_CLOUD_TOKEN_JSON }} | |
| create_credentials_file: true | |
| cleanup_credentials: true | |
| - name: Run project on kubeflow pipelines | |
| working-directory: spaceflights | |
| run: | | |
| kedro kubeflow upload-pipeline | |
| kedro kubeflow run-once --wait-for-completion --timeout 1200 | |
| env: | |
| KEDRO_CONFIG_KUBEFLOW_HOSTNAME: ${{ secrets.KUBEFLOW_HOSTNAME }} | |
| KEDRO_CONFIG_COMMIT_ID: ${{ github.sha }} | |
| KEDRO_CONFIG_RUN_ID: ${{ github.run_id }} |