From 755749848038bb1c0da253affd0b81c88f293c07 Mon Sep 17 00:00:00 2001 From: Ankita Katiyar <110245118+ankatiyar@users.noreply.github.com> Date: Mon, 4 Sep 2023 18:56:29 +0100 Subject: [PATCH] ci: Refactor the workflows (#322) --- .github/workflows/check-plugin.yml | 183 -------------------------- .github/workflows/check-release.yml | 31 ++++- .github/workflows/e2e-tests.yml | 43 ++++++ .github/workflows/kedro-airflow.yml | 34 ++++- .github/workflows/kedro-datasets.yml | 47 ++++++- .github/workflows/kedro-docker.yml | 34 ++++- .github/workflows/kedro-telemetry.yml | 34 ++++- .github/workflows/lint.yml | 44 +++++++ .github/workflows/nightly-build.yml | 92 +++++++++++++ .github/workflows/unit-tests.yml | 63 +++++++++ 10 files changed, 391 insertions(+), 214 deletions(-) delete mode 100644 .github/workflows/check-plugin.yml create mode 100644 .github/workflows/e2e-tests.yml create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/nightly-build.yml create mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/check-plugin.yml b/.github/workflows/check-plugin.yml deleted file mode 100644 index e0df1114c..000000000 --- a/.github/workflows/check-plugin.yml +++ /dev/null @@ -1,183 +0,0 @@ -name: Running tests and linter - -on: - workflow_call: - inputs: - plugin: - type: string - -jobs: - - setup-matrix: - # kedro-datasets is the only plugin that supports python 3.11 - runs-on: ubuntu-latest - outputs: - python-versions: ${{ steps.set-matrix.outputs.matrix }} - steps: - - id: set-matrix - run: | - if [[ "${{ inputs.plugin }}" == "kedro-datasets" ]]; then - MATRIX='["3.7", "3.8", "3.9", "3.10", "3.11"]' - else - MATRIX='["3.7", "3.8", "3.9", "3.10"]' - fi - echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT - - unit-tests: - needs: setup-matrix - defaults: - run: - shell: bash - strategy: - matrix: - os: [ ubuntu-latest, windows-latest ] - python-version: ${{fromJson(needs.setup-matrix.outputs.python-versions)}} - runs-on: ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Python ${{matrix.python-version}} - uses: actions/setup-python@v3 - with: - python-version: ${{matrix.python-version}} - - name: Cache python packages for Linux - if: matrix.os == 'ubuntu-latest' - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} - restore-keys: ${{inputs.plugin}} - - name: Cache python packages for Windows - if: matrix.os == 'windows-latest' - uses: actions/cache@v3 - with: - path: ~\AppData\Local\pip\Cache - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} - restore-keys: ${{inputs.plugin}} - - name: Install Kedro - run: pip install git+https://github.com/kedro-org/kedro@main - - name: Add MSBuild to PATH - if: matrix.os == 'windows-latest' - uses: microsoft/setup-msbuild@v1 - - name: Install dependencies - run: | - cd ${{ inputs.plugin }} - pip install ".[test]" - - name: pip freeze - run: pip freeze - - name: Run unit tests for Linux / all plugins - if: matrix.os != 'windows-latest' - run: make plugin=${{ inputs.plugin }} test - - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry - if: matrix.os == 'windows-latest' && inputs.plugin != 'kedro-datasets' - run: | - cd ${{ inputs.plugin }} - pytest tests - - name: Run unit tests for Windows / kedro-datasets / no spark parallel - if: matrix.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' - run: | - make test-no-spark - - lint: - defaults: - run: - shell: bash - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - # kedro-datasets is the only plugin that supports python 3.11 - - name: Determine Python version for linting - id: get-python-version - run: | - if [[ "${{ inputs.plugin }}" == "kedro-datasets" ]]; then - echo "version=3.11" >> $GITHUB_OUTPUT - else - echo "version=3.8" >> $GITHUB_OUTPUT - fi - - name: Set up Python - uses: actions/setup-python@v3 - with: - python-version: ${{ steps.get-python-version.outputs.version }} - - name: Cache python packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} - restore-keys: ${{inputs.plugin}} - - name: Install dependencies - run: | - cd ${{ inputs.plugin }} - pip install git+https://github.com/kedro-org/kedro@main - pip install ".[test]" - pip freeze - - name: Install pre-commit hooks - run: | - pre-commit install --install-hooks - pre-commit install --hook-type pre-push - - name: Run linter - run: make plugin=${{ inputs.plugin }} lint - - RTD-build: - if: inputs.plugin == 'kedro-datasets' - defaults: - run: - shell: bash - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Python 3.8 - uses: actions/setup-python@v3 - with: - python-version: "3.8" - - name: Cache python packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{inputs.plugin}}-ubuntu-latest-python-"3.8" - restore-keys: ${{inputs.plugin}} - - name: Install dependencies - run: | - cd ${{ inputs.plugin }} - pip install ".[docs]" - pip install ".[test]" - - name: RTD build for kedro-datasets - run: | - make rtd - - e2e-tests: - if: inputs.plugin != 'kedro-datasets' - defaults: - run: - shell: bash - strategy: - matrix: - os: [ ubuntu-latest ] - python-version: [ "3.7", "3.8", "3.9", "3.10" ] - runs-on: ${{ matrix.os }} - steps: - - name: Checkout code - uses: actions/checkout@v3 - - name: Set up Python ${{matrix.python-version}} - uses: actions/setup-python@v3 - with: - python-version: ${{matrix.python-version}} - - name: Cache python packages - uses: actions/cache@v3 - with: - path: ~/.cache/pip - key: ${{inputs.plugin}}-${{matrix.os}}-python-${{matrix.python-version}} - restore-keys: ${{inputs.plugin}} - - name: Install dependencies - run: | - cd ${{ inputs.plugin }} - pip install git+https://github.com/kedro-org/kedro@main - pip install ".[test]" - - name: pip freeze - run: pip freeze - - name: Run end to end tests - # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run` - # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241 - shell: 'script -q -e -c "bash {0}"' - run: make plugin=${{ inputs.plugin }} e2e-tests diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 51036d260..06d0862fe 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -31,16 +31,33 @@ jobs: package_name: ${{ steps.version_check.outputs.package_name }} package_version: ${{ steps.version_check.outputs.package_version }} - test: + airflow-test: needs: check-version - if: ${{ needs.check-version.outputs.new_release == 'true' }} - uses: ./.github/workflows/check-plugin.yml - with: - plugin: ${{ needs.check-version.outputs.package_name }} + if: ${{ needs.check-version.outputs.new_release == 'true' && needs.check-version.outputs.package_name == 'kedro-airflow' }} + uses: ./.github/workflows/kedro-airflow.yml + + datasets-test: + needs: check-version + if: ${{ needs.check-version.outputs.new_release == 'true' && needs.check-version.outputs.package_name == 'kedro-datasets' }} + uses: ./.github/workflows/kedro-datasets.yml + + docker-test: + needs: check-version + if: ${{ needs.check-version.outputs.new_release == 'true' && needs.check-version.outputs.package_name == 'kedro-docker' }} + uses: ./.github/workflows/kedro-docker.yml + + telemetry-test: + needs: check-version + if: ${{ needs.check-version.outputs.new_release == 'true' && needs.check-version.outputs.package_name == 'kedro-telemetry' }} + uses: ./.github/workflows/kedro-telemetry.yml build-publish: - needs: [check-version, test] - if: ${{ needs.check-version.outputs.new_release == 'true' }} + needs: [ check-version, airflow-test, docker-test, datasets-test, telemetry-test ] + if: | + always() && + !contains(needs.*.result, 'failure') && + !contains(needs.*.result, 'cancelled') && + needs.check-version.outputs.new_release == 'true' runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml new file mode 100644 index 000000000..beb629b28 --- /dev/null +++ b/.github/workflows/e2e-tests.yml @@ -0,0 +1,43 @@ +name: Run e2e tests + +on: + workflow_call: + inputs: + plugin: + type: string + os: + type: string + python-version: + type: string + +jobs: + e2e-tests: + defaults: + run: + shell: bash + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{inputs.python-version}} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install git+https://github.com/kedro-org/kedro@main + pip install ".[test]" + - name: pip freeze + run: pip freeze + - name: Run end to end tests + # Custom shell to run kedro-docker e2e-tests because -it flag for `docker run` + # isn't supported on Github Actions. See https://github.com/actions/runner/issues/241 + shell: 'script -q -e -c "bash {0}"' + run: make plugin=${{ inputs.plugin }} e2e-tests diff --git a/.github/workflows/kedro-airflow.yml b/.github/workflows/kedro-airflow.yml index fecb91db2..9b7e95c33 100644 --- a/.github/workflows/kedro-airflow.yml +++ b/.github/workflows/kedro-airflow.yml @@ -1,6 +1,7 @@ -name: Run checks on Kedro-Airflow +name: Run checks on kedro-airflow on: + workflow_call: push: branches: - main @@ -15,12 +16,33 @@ on: - "kedro-datasets/**" - "kedro-docker/**" - "kedro-telemetry/**" - schedule: - # Run every day at midnight (UTC time) - - cron: '0 0 * * *' jobs: - airflow-test: - uses: ./.github/workflows/check-plugin.yml + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-airflow + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + uses: ./.github/workflows/lint.yml + with: + plugin: kedro-airflow + os: ubuntu-latest + python-version: "3.8" + + e2e-tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/e2e-tests.yml + with: + plugin: kedro-airflow + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/kedro-datasets.yml b/.github/workflows/kedro-datasets.yml index be4315cd9..c4cf02cfb 100644 --- a/.github/workflows/kedro-datasets.yml +++ b/.github/workflows/kedro-datasets.yml @@ -1,6 +1,7 @@ -name: Run checks on Kedro-Datasets +name: Run checks on kedro-datasets on: + workflow_call: push: branches: - main @@ -15,12 +16,46 @@ on: - "kedro-airflow/**" - "kedro-docker/**" - "kedro-telemetry/**" - schedule: - # Run every day at midnight (UTC time) - - cron: '0 0 * * *' jobs: - datasets-test: - uses: ./.github/workflows/check-plugin.yml + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10", "3.11" ] + uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-datasets + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + uses: ./.github/workflows/lint.yml + with: + plugin: kedro-datasets + os: ubuntu-latest + python-version: "3.11" + + RTD-build: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python 3.8 + uses: actions/setup-python@v3 + with: + python-version: "3.8" + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: kedro-datasets-ubuntu-latest-python-"3.8" + restore-keys: kedro-datasets + - name: Install dependencies + run: | + cd kedro-datasets + pip install ".[docs]" + pip install ".[test]" + - name: RTD build for kedro-datasets + run: | + make rtd diff --git a/.github/workflows/kedro-docker.yml b/.github/workflows/kedro-docker.yml index 430058513..0522dd0a6 100644 --- a/.github/workflows/kedro-docker.yml +++ b/.github/workflows/kedro-docker.yml @@ -1,6 +1,7 @@ -name: Run checks on Kedro-Docker +name: Run checks on kedro-docker on: + workflow_call: push: branches: - main @@ -15,12 +16,33 @@ on: - "kedro-airflow/**" - "kedro-datasets/**" - "kedro-telemetry/**" - schedule: - # Run every day at midnight (UTC time) - - cron: '0 0 * * *' jobs: - docker-test: - uses: ./.github/workflows/check-plugin.yml + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest, windows-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-docker + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + uses: ./.github/workflows/lint.yml + with: + plugin: kedro-docker + os: ubuntu-latest + python-version: "3.8" + + e2e-tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/e2e-tests.yml + with: + plugin: kedro-docker + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/kedro-telemetry.yml b/.github/workflows/kedro-telemetry.yml index 88fb8ae7a..ae5e0b1c0 100644 --- a/.github/workflows/kedro-telemetry.yml +++ b/.github/workflows/kedro-telemetry.yml @@ -1,6 +1,7 @@ -name: Run checks on Kedro-Telemetry +name: Run checks on kedro-telemetry on: + workflow_call: push: branches: - main @@ -15,12 +16,33 @@ on: - "kedro-airflow/**" - "kedro-datasets/**" - "kedro-docker/**" - schedule: - # Run every day at midnight (UTC time) - - cron: '0 0 * * *' jobs: - telemetry-test: - uses: ./.github/workflows/check-plugin.yml + unit-tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/unit-tests.yml with: plugin: kedro-telemetry + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} + + lint: + uses: ./.github/workflows/lint.yml + with: + plugin: kedro-telemetry + os: ubuntu-latest + python-version: "3.8" + + e2e-tests: + strategy: + matrix: + os: [ ubuntu-latest ] + python-version: [ "3.7", "3.8", "3.9", "3.10" ] + uses: ./.github/workflows/e2e-tests.yml + with: + plugin: kedro-telemetry + os: ${{ matrix.os }} + python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 000000000..fe83fed95 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,44 @@ +name: Run linter + +on: + workflow_call: + inputs: + plugin: + type: string + os: + type: string + python-version: + type: string + +jobs: + lint: + defaults: + run: + shell: bash + runs-on: ${{ inputs.os }} + steps: + - name: Checkout code + uses: actions/checkout@v3 + # kedro-datasets is the only plugin that supports python 3.11 + - name: Set up Python + uses: actions/setup-python@v3 + with: + python-version: ${{ inputs.python-version }} + - name: Cache python packages + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install git+https://github.com/kedro-org/kedro@main + pip install ".[test]" + pip freeze + - name: Install pre-commit hooks + run: | + pre-commit install --install-hooks + pre-commit install --hook-type pre-push + - name: Run linter + run: make plugin=${{ inputs.plugin }} lint diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml new file mode 100644 index 000000000..234dec8a1 --- /dev/null +++ b/.github/workflows/nightly-build.yml @@ -0,0 +1,92 @@ +name: Run nightly tests on all plugins + +on: + workflow_dispatch: + schedule: + # Run every day at midnight (UTC time) + - cron: '0 0 * * *' + +jobs: + airflow-test: + uses: ./.github/workflows/kedro-airflow.yml + + datasets-test: + uses: ./.github/workflows/kedro-datasets.yml + + docker-test: + uses: ./.github/workflows/kedro-docker.yml + + telemetry-test: + uses: ./.github/workflows/kedro-telemetry.yml + + notify-airflow: + permissions: + issues: write + name: Notify failed build for airflow + needs: airflow-test + if: failure() + runs-on: ubuntu-latest + steps: + - uses: jayqi/failed-build-issue-action@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + label-name: "airflow nightly build" + title-template: "`kedro-airflow`: Nightly build failure" + body-template: | + GitHub Actions workflow [{{workflow}} #{{runNumber}}](https://github.com/{{repo.owner}}/{{repo.repo}}/actions/runs/{{runId}}) failed. + create-label: true + always-create-new-issue: false + + notify-datasets: + permissions: + issues: write + name: Notify failed build for datasets + needs: datasets-test + if: failure() + runs-on: ubuntu-latest + steps: + - uses: jayqi/failed-build-issue-action@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + label-name: "datasets nightly build" + title-template: "`kedro-datasets`: Nightly build failure" + body-template: | + GitHub Actions workflow [{{workflow}} #{{runNumber}}](https://github.com/{{repo.owner}}/{{repo.repo}}/actions/runs/{{runId}}) failed. + create-label: true + always-create-new-issue: false + + notify-docker: + permissions: + issues: write + name: Notify failed build for docker + needs: docker-test + if: failure() + runs-on: ubuntu-latest + steps: + - uses: jayqi/failed-build-issue-action@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + label-name: "docker nightly build" + title-template: "`kedro-docker`: Nightly build failure" + body-template: | + GitHub Actions workflow [{{workflow}} #{{runNumber}}](https://github.com/{{repo.owner}}/{{repo.repo}}/actions/runs/{{runId}}) failed. + create-label: true + always-create-new-issue: false + + notify-telemetry: + permissions: + issues: write + name: Notify failed build for telemetry + needs: telemetry-test + if: failure() + runs-on: ubuntu-latest + steps: + - uses: jayqi/failed-build-issue-action@v1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + label-name: "telemetry nightly build" + title-template: "`kedro-telemetry`: Nightly build failure" + body-template: | + GitHub Actions workflow [{{workflow}} #{{runNumber}}](https://github.com/{{repo.owner}}/{{repo.repo}}/actions/runs/{{runId}}) failed. + create-label: true + always-create-new-issue: false diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml new file mode 100644 index 000000000..5f479afa5 --- /dev/null +++ b/.github/workflows/unit-tests.yml @@ -0,0 +1,63 @@ +name: Run unit tests + +on: + workflow_call: + inputs: + plugin: + type: string + os: + type: string + python-version: + type: string + +jobs: + + unit-tests: + runs-on: ${{ inputs.os }} + defaults: + run: + shell: bash + steps: + - name: Checkout code + uses: actions/checkout@v3 + - name: Set up Python ${{inputs.python-version}} + uses: actions/setup-python@v3 + with: + python-version: ${{inputs.python-version}} + - name: Cache python packages for Linux + if: inputs.os == 'ubuntu-latest' + uses: actions/cache@v3 + with: + path: ~/.cache/pip + key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Cache python packages for Windows + if: inputs.os == 'windows-latest' + uses: actions/cache@v3 + with: + path: ~\AppData\Local\pip\Cache + key: ${{inputs.plugin}}-${{inputs.os}}-python-${{inputs.python-version}} + restore-keys: ${{inputs.plugin}} + - name: Install Kedro + run: pip install git+https://github.com/kedro-org/kedro@main + - name: Add MSBuild to PATH + if: inputs.os == 'windows-latest' + uses: microsoft/setup-msbuild@v1 + - name: Install dependencies + run: | + cd ${{ inputs.plugin }} + pip install ".[test]" + - name: pip freeze + run: pip freeze + - name: Run unit tests for Linux / all plugins + if: inputs.os != 'windows-latest' + run: make plugin=${{ inputs.plugin }} test + - name: Run unit tests for Windows / kedro-airflow, kedro-docker, kedro-telemetry + if: inputs.os == 'windows-latest' && inputs.plugin != 'kedro-datasets' + run: | + cd ${{ inputs.plugin }} + pytest tests + - name: Run unit tests for Windows / kedro-datasets / no spark parallel + if: inputs.os == 'windows-latest' && inputs.plugin == 'kedro-datasets' + run: | + make test-no-spark