[DO NOT MERGE] Test RC provider packages for https://github.com/apache/airflow/issues/41080 #5908
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Build and test astro Python SDK | |
on: | |
push: | |
branches: [ 'main', 'release-**' ] | |
paths: | |
- 'python-sdk/**' | |
- '.github/workflows/ci-python-sdk.yaml' | |
pull_request: | |
branches: [ 'main', 'release-**' ] | |
paths: | |
- 'python-sdk/**' | |
- '.github/workflows/ci-python-sdk.yaml' | |
- '*' | |
# Run on PRs from forks | |
pull_request_target: | |
branches: [ 'main' ] | |
types: ['labeled'] | |
paths: | |
- 'python-sdk/**' | |
- '.github/workflows/ci-python-sdk.yaml' | |
- '*' | |
release: | |
types: [ 'created' ] | |
defaults: | |
run: | |
working-directory: python-sdk | |
# This allows a subsequently queued workflow run to interrupt and cancel previous runs | |
concurrency: | |
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
cancel-in-progress: true | |
# This list should only have non-sensitive env vars | |
# Env vars with secrets should be in the specific jobs | |
env: | |
SETUPTOOLS_USE_DISTUTILS: stdlib | |
POSTGRES_HOST: postgres | |
POSTGRES_PORT: 5432 | |
AIRFLOW__ASTRO_SDK__SQL_SCHEMA: astroflow_ci | |
REDSHIFT_DATABASE: dev | |
REDSHIFT_HOST: utkarsh-cluster.cdru7mxqmtyx.us-east-2.redshift.amazonaws.com | |
SNOWFLAKE_SCHEMA: ASTROFLOW_CI | |
SNOWFLAKE_DATABASE: SANDBOX | |
SNOWFLAKE_WAREHOUSE: DEMO | |
SNOWFLAKE_HOST: https://gp21411.us-east-1.snowflakecomputing.com | |
SNOWFLAKE_ACCOUNT: gp21411 | |
SNOWFLAKE_REGION: us-east-1 | |
SNOWFLAKE_ROLE: AIRFLOW_TEST_USER | |
SFTP_HOSTNAME: ${{ secrets.SFTP_HOSTNAME }} | |
SFTP_USERNAME: ${{ secrets.SFTP_USERNAME }} | |
SFTP_PASSWORD: ${{ secrets.SFTP_PASSWORD }} | |
AIRFLOW__CORE__LOAD_DEFAULT_CONNECTIONS: True | |
AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} | |
AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }} | |
AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }} | |
AIRFLOW_VAR_FOO: templated_file_name | |
AWS_BUCKET: tmp9 | |
GOOGLE_BUCKET: dag-authoring | |
FORCE_COLOR: "true" | |
MSSQL_DB: ${{ secrets.MSSQL_DB }} | |
MSSQL_HOST: ${{ secrets.MSSQL_HOST }} | |
MSSQL_LOGIN: ${{ secrets.MSSQL_LOGIN }} | |
MSSQL_PASSWORD: ${{ secrets.MSSQL_PASSWORD }} | |
MYSQL_DB: ${{ secrets.MYSQL_DB }} | |
MYSQL_HOST: ${{ secrets.MYSQL_HOST }} | |
MYSQL_LOGIN: ${{ secrets.MYSQL_LOGIN }} | |
MYSQL_PASSWORD: ${{ secrets.MYSQL_PASSWORD }} | |
jobs: | |
Markdown-link-check: | |
if: github.event.action != 'labeled' | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: gaurav-nelson/github-action-markdown-link-check@v1 | |
with: | |
config-file: '.github/workflows/mlc_config.json' | |
Type-Check: | |
if: github.event.action != 'labeled' | |
runs-on: ubuntu-latest | |
env: | |
MYPY_FORCE_COLOR: 1 | |
TERM: xterm-color | |
SETUPTOOLS_USE_DISTUTILS: stdlib | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v3 | |
with: | |
python-version: '3.10' | |
architecture: 'x64' | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cache/pip | |
.nox | |
key: ${{ runner.os }}-${{ hashFiles('python-sdk/pyproject.toml') }} | |
- run: | | |
sudo apt update | |
sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
- run: pip3 install nox | |
- run: nox -s type_check | |
Build-Docs: | |
if: github.event.action != 'labeled' | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v3 | |
with: | |
python-version: '3.10' | |
architecture: 'x64' | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cache/pip | |
.nox | |
key: ${{ runner.os }}-${{ hashFiles('python-sdk/pyproject.toml') }} | |
- run: | | |
sudo apt update | |
sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
- run: pip3 install nox | |
- run: nox -s build_docs | |
Run-Optional-Packages-tests-python-sdk: | |
if: >- | |
github.event_name == 'push' || | |
( | |
github.event_name == 'pull_request' && | |
github.event.pull_request.head.repo.fork == false | |
) || | |
( | |
github.event_name == 'pull_request_target' && | |
contains(github.event.pull_request.labels.*.name, 'safe to test') | |
) || | |
( | |
github.event_name == 'release' | |
) | |
runs-on: ubuntu-latest | |
services: | |
postgres: | |
image: dimberman/pagila-test | |
env: | |
POSTGRES_PASSWORD: postgres | |
options: >- | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
--name postgres | |
ports: | |
- 5432:5432 | |
sftp: | |
image: ghcr.io/astronomer/astro-sdk/sftp_docker | |
ports: | |
- 2222:22 | |
ftp: | |
image: ghcr.io/astronomer/astro-sdk/ftp_docker | |
ports: | |
- 21:21 | |
- 30000-30009:30000-30009 | |
env: | |
FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }} | |
FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }} | |
FTP_USER_HOME: /home/foo | |
PUBLICHOST: "localhost" | |
steps: | |
- uses: actions/checkout@v3 | |
if: github.event_name != 'pull_request_target' | |
- name: Checkout pull/${{ github.event.number }} | |
uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.event.pull_request.head.sha }} | |
if: github.event_name == 'pull_request_target' | |
- uses: actions/setup-python@v3 | |
with: | |
python-version: '3.10' | |
architecture: 'x64' | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cache/pip | |
.nox | |
key: ${{ runner.os }}-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }} | |
- run: cat ../.github/ci-test-connections.yaml > test-connections.yaml | |
- run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} | |
- run: sqlite3 /tmp/sqlite_default.db "VACUUM;" | |
- run: | | |
sudo apt update | |
sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
- run: pip3 install nox | |
- run: nox -s test_examples_by_dependency -- --cov=src --cov-report=xml --cov-branch | |
- name: Upload coverage | |
uses: actions/upload-artifact@v2 | |
with: | |
name: coverage${{ matrix.group }} | |
path: ./python-sdk/.coverage | |
env: | |
GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }} | |
GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }} | |
REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }} | |
REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }} | |
SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }} | |
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} | |
DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }} | |
AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} | |
AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }} | |
AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }} | |
Run-Unit-tests-Airflow-2-8: | |
strategy: | |
matrix: | |
version: [ '3.8', '3.9', '3.10', '3.11' ] | |
if: >- | |
github.event_name == 'push' || | |
( | |
github.event_name == 'pull_request' && | |
github.event.pull_request.head.repo.fork == false | |
) || | |
( | |
github.event_name == 'pull_request_target' && | |
contains(github.event.pull_request.labels.*.name, 'safe to test') | |
) || | |
( | |
github.event_name == 'release' | |
) | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
if: github.event_name != 'pull_request_target' | |
- name: Checkout pull/${{ github.event.number }} | |
uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.event.pull_request.head.sha }} | |
if: github.event_name == 'pull_request_target' | |
- uses: actions/setup-python@v3 | |
with: | |
python-version: ${{ matrix.version }} | |
architecture: 'x64' | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cache/pip | |
.nox | |
key: ${{ runner.os }}-2.8-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }} | |
- run: sqlite3 /tmp/sqlite_default.db "VACUUM;" | |
- run: | | |
sudo apt update | |
sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
- run: pip3 install nox | |
- run: nox -s "test-${{ matrix.version }}(airflow='2.8')" -- tests/ --cov=src --cov-report=xml --cov-branch | |
- name: Upload coverage | |
uses: actions/upload-artifact@v2 | |
with: | |
name: coverage-unit-test | |
path: ./python-sdk/.coverage | |
Run-load-file-Integration-Airflow-2-8: | |
if: >- | |
github.event_name == 'push' || | |
( | |
github.event_name == 'pull_request' && | |
github.event.pull_request.head.repo.fork == false | |
) || | |
( | |
github.event_name == 'pull_request_target' && | |
contains(github.event.pull_request.labels.*.name, 'safe to test') | |
) || | |
( | |
github.event_name == 'release' | |
) | |
strategy: | |
fail-fast: false | |
matrix: | |
group: [ 1, 2, 3] | |
runs-on: ubuntu-latest | |
services: | |
postgres: | |
# Docker Hub image | |
image: dimberman/pagila-test | |
env: | |
POSTGRES_PASSWORD: postgres | |
# Set health checks to wait until postgres has started | |
options: >- | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
ports: | |
- 5432:5432 | |
sftp: | |
image: ghcr.io/astronomer/astro-sdk/sftp_docker | |
ports: | |
- 2222:22 | |
ftp: | |
image: ghcr.io/astronomer/astro-sdk/ftp_docker | |
ports: | |
- 21:21 | |
- 30000-30009:30000-30009 | |
env: | |
FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }} | |
FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }} | |
FTP_USER_HOME: /home/foo | |
PUBLICHOST: "localhost" | |
steps: | |
- uses: actions/checkout@v3 | |
if: github.event_name != 'pull_request_target' | |
- name: Checkout pull/${{ github.event.number }} | |
uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.event.pull_request.head.sha }} | |
if: github.event_name == 'pull_request_target' | |
- uses: actions/setup-python@v3 | |
with: | |
python-version: '3.10' | |
architecture: 'x64' | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cache/pip | |
.nox | |
key: ${{ runner.os }}-2.5-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }} | |
- run: cat ../.github/ci-test-connections.yaml > test-connections.yaml | |
- run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} | |
- run: sqlite3 /tmp/sqlite_default.db "VACUUM;" | |
- run: | | |
sudo apt update | |
sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
- run: pip3 install nox | |
- run: nox -s "test-3.10(airflow='2.8')" -- tests_integration/ -k "test_load_file.py and not redshift" --splits 3 --group ${{ matrix.group }} --store-durations --durations-path /tmp/durations-${{ matrix.group }} --cov=src --cov-report=xml --cov-branch | |
- run: cat /tmp/durations-${{ matrix.group }} | |
- name: Upload coverage | |
uses: actions/upload-artifact@v2 | |
with: | |
name: coverage-${{ matrix.group }}-integration-tests | |
path: ./python-sdk/.coverage | |
- name: Collect pytest durations | |
uses: actions/upload-artifact@v2 | |
with: | |
name: pytest_durations_load_file_${{ matrix.group }} | |
path: /tmp/durations-${{ matrix.group }} | |
env: | |
GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }} | |
GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }} | |
REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }} | |
REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }} | |
SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }} | |
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} | |
DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }} | |
AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} | |
AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }} | |
Run-example-dag-Integration-Airflow-2-8: | |
if: >- | |
github.event_name == 'push' || | |
( | |
github.event_name == 'pull_request' && | |
github.event.pull_request.head.repo.fork == false | |
) || | |
( | |
github.event_name == 'pull_request_target' && | |
contains(github.event.pull_request.labels.*.name, 'safe to test') | |
) || | |
( | |
github.event_name == 'release' | |
) | |
strategy: | |
fail-fast: false | |
matrix: | |
group: [ 1, 2, 3] | |
runs-on: ubuntu-latest | |
services: | |
postgres: | |
# Docker Hub image | |
image: dimberman/pagila-test | |
env: | |
POSTGRES_PASSWORD: postgres | |
# Set health checks to wait until postgres has started | |
options: >- | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
ports: | |
- 5432:5432 | |
sftp: | |
image: ghcr.io/astronomer/astro-sdk/sftp_docker | |
ports: | |
- 2222:22 | |
ftp: | |
image: ghcr.io/astronomer/astro-sdk/ftp_docker | |
ports: | |
- 21:21 | |
- 30000-30009:30000-30009 | |
env: | |
FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }} | |
FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }} | |
FTP_USER_HOME: /home/foo | |
PUBLICHOST: "localhost" | |
steps: | |
- uses: actions/checkout@v3 | |
if: github.event_name != 'pull_request_target' | |
- name: Checkout pull/${{ github.event.number }} | |
uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.event.pull_request.head.sha }} | |
if: github.event_name == 'pull_request_target' | |
- uses: actions/setup-python@v3 | |
with: | |
python-version: '3.10' | |
architecture: 'x64' | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cache/pip | |
.nox | |
key: ${{ runner.os }}-2.8-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }} | |
- run: cat ../.github/ci-test-connections.yaml > test-connections.yaml | |
- run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} | |
- run: sqlite3 /tmp/sqlite_default.db "VACUUM;" | |
- run: | | |
sudo apt update | |
sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
- run: pip3 install nox | |
- run: nox -s "test-3.10(airflow='2.8')" -- tests_integration/ -k "test_example_dags.py and not redshift" --splits 3 --group ${{ matrix.group }} --store-durations --durations-path /tmp/durations-${{ matrix.group }} --cov=src --cov-report=xml --cov-branch | |
- run: cat /tmp/durations-${{ matrix.group }} | |
- name: Upload coverage | |
uses: actions/upload-artifact@v2 | |
with: | |
name: coverage-${{ matrix.group }}-integration-tests | |
path: ./python-sdk/.coverage | |
- name: Collect pytest durations | |
uses: actions/upload-artifact@v2 | |
with: | |
name: pytest_durations_example_dags_${{ matrix.group }} | |
path: /tmp/durations-${{ matrix.group }} | |
env: | |
GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }} | |
GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }} | |
REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }} | |
REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }} | |
SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }} | |
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} | |
DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }} | |
AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} | |
AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }} | |
Run-Integration-tests-Airflow-2-8: | |
if: >- | |
github.event_name == 'push' || | |
( | |
github.event_name == 'pull_request' && | |
github.event.pull_request.head.repo.fork == false | |
) || | |
( | |
github.event_name == 'pull_request_target' && | |
contains(github.event.pull_request.labels.*.name, 'safe to test') | |
) || | |
( | |
github.event_name == 'release' | |
) | |
strategy: | |
fail-fast: false | |
matrix: | |
group: [ 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11 ] | |
runs-on: ubuntu-latest | |
services: | |
postgres: | |
# Docker Hub image | |
image: dimberman/pagila-test | |
env: | |
POSTGRES_PASSWORD: postgres | |
# Set health checks to wait until postgres has started | |
options: >- | |
--health-cmd pg_isready | |
--health-interval 10s | |
--health-timeout 5s | |
--health-retries 5 | |
ports: | |
- 5432:5432 | |
sftp: | |
image: ghcr.io/astronomer/astro-sdk/sftp_docker | |
ports: | |
- 2222:22 | |
ftp: | |
image: ghcr.io/astronomer/astro-sdk/ftp_docker | |
ports: | |
- 21:21 | |
- 30000-30009:30000-30009 | |
env: | |
FTP_USER_NAME: ${{ secrets.SFTP_USERNAME }} | |
FTP_USER_PASS: ${{ secrets.SFTP_PASSWORD }} | |
FTP_USER_HOME: /home/foo | |
PUBLICHOST: "localhost" | |
steps: | |
- uses: actions/checkout@v3 | |
if: github.event_name != 'pull_request_target' | |
- name: Checkout pull/${{ github.event.number }} | |
uses: actions/checkout@v3 | |
with: | |
ref: ${{ github.event.pull_request.head.sha }} | |
if: github.event_name == 'pull_request_target' | |
- uses: actions/setup-python@v3 | |
with: | |
python-version: '3.10' | |
architecture: 'x64' | |
- uses: actions/cache@v3 | |
with: | |
path: | | |
~/.cache/pip | |
.nox | |
key: ${{ runner.os }}-2.8-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }} | |
- run: cat ../.github/ci-test-connections.yaml > test-connections.yaml | |
- run: python -c 'import os; print(os.getenv("GOOGLE_APPLICATION_CREDENTIALS_JSON", "").strip())' > ${{ env.GOOGLE_APPLICATION_CREDENTIALS }} | |
- run: sqlite3 /tmp/sqlite_default.db "VACUUM;" | |
- run: | | |
sudo apt update | |
sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
- run: pip3 install nox | |
- run: nox -s "test-3.10(airflow='2.8')" -- tests_integration/ -k "not test_load_file.py and not test_example_dags.py and not redshift" --splits 11 --group ${{ matrix.group }} --store-durations --durations-path /tmp/durations-${{ matrix.group }} --cov=src --cov-report=xml --cov-branch | |
- run: cat /tmp/durations-${{ matrix.group }} | |
- name: Upload coverage | |
uses: actions/upload-artifact@v2 | |
with: | |
name: coverage-${{ matrix.group }}-integration-tests | |
path: ./python-sdk/.coverage | |
- name: Collect pytest durations | |
uses: actions/upload-artifact@v2 | |
with: | |
name: pytest_durations_integration_tests_${{ matrix.group }} | |
path: /tmp/durations-${{ matrix.group }} | |
env: | |
GOOGLE_APPLICATION_CREDENTIALS_JSON: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS_JSON }} | |
GOOGLE_APPLICATION_CREDENTIALS: /tmp/google_credentials.json | |
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN: ${{ secrets.REDSHIFT_NATIVE_LOAD_IAM_ROLE_ARN }} | |
REDSHIFT_USERNAME: ${{ secrets.REDSHIFT_USERNAME }} | |
REDSHIFT_PASSWORD: ${{ secrets.REDSHIFT_PASSWORD }} | |
SNOWFLAKE_ACCOUNT_NAME: ${{ secrets.SNOWFLAKE_UNAME }} | |
SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} | |
DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} | |
DATABRICKS_HOST: ${{ secrets.DATABRICKS_HOST }} | |
DATABRICKS_HTTP_PATH: ${{ secrets.DATABRICKS_HTTP_PATH }} | |
AIRFLOW__ASTRO_SDK__DATABRICKS_CLUSTER_ID: ${{ secrets.DATABRICKS_CLUSTER_ID }} | |
AZURE_WASB_CONN_STRING: ${{ secrets.AZURE_WASB_CONN_STRING }} | |
AZURE_WASB_ACCESS_KEY: ${{ secrets.AZURE_WASB_ACCESS_KEY }} | |
Generate-Constraints: | |
if: (github.event_name == 'release' || github.event_name == 'push') | |
strategy: | |
fail-fast: false | |
matrix: | |
python: [ '3.8', '3.9', '3.10', '3.11' ] | |
airflow: [ '2.7', '2.8'] | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- uses: actions/setup-python@v3 | |
with: | |
python-version: '${{ matrix.python }}' | |
architecture: 'x64' | |
- name: Get pip cache dir | |
id: pip-cache | |
run: | | |
echo "pip_cache_dir=$(pip cache dir)" >> $GITHUB_OUTPUT | |
- uses: actions/cache@v3 | |
with: | |
path: ${{ steps.pip-cache.outputs.pip_cache_dir }} | |
key: constraints-${{ matrix.python }}-${{ matrix.airflow }}-${{ hashFiles('python-sdk/pyproject.toml') }}-${{ hashFiles('python-sdk/src/astro/__init__.py') }} | |
- run: pip3 install -U pip wheel build nox | |
- run: nox -s build | |
- run: pip3 install 'apache-airflow~=${{ matrix.airflow }}' "$(ls dist/*.whl)[all]" | |
- run: pip3 list --format=freeze > constraints-${{ matrix.python }}-${{ matrix.airflow }} | |
- run: cat constraints-${{ matrix.python }}-${{ matrix.airflow }} | |
- name: Upload constraints | |
uses: actions/upload-artifact@v3 | |
with: | |
name: constraints-${{ matrix.python }}-${{ matrix.airflow }} | |
path: ./python-sdk/constraints-${{ matrix.python }}-${{ matrix.airflow }} | |
if-no-files-found: error | |
Code-Coverage: | |
if: github.event.action != 'labeled' | |
needs: | |
- Run-Unit-tests-Airflow-2-8 | |
- Run-Integration-tests-Airflow-2-8 | |
- Run-load-file-Integration-Airflow-2-8 | |
- Run-example-dag-Integration-Airflow-2-8 | |
- Run-Optional-Packages-tests-python-sdk | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Set up Python 3.10 | |
uses: actions/setup-python@v3 | |
with: | |
python-version: '3.10' | |
- name: Install coverage | |
run: | | |
pip3 install coverage | |
- name: Download all artifacts | |
uses: actions/download-artifact@v2 | |
with: | |
path: ./python-sdk/coverage | |
- name: Run coverage | |
run: | | |
coverage combine ./coverage/coverage*/.coverage | |
coverage report | |
coverage xml | |
- name: Upload coverage | |
uses: codecov/codecov-action@v2 | |
with: | |
fail_ci_if_error: true | |
token: ${{ secrets.CODECOV_TOKEN }} | |
flags: PythonSDK | |
files: ./python-sdk/coverage.xml | |
collect-durations: | |
runs-on: ubuntu-latest | |
needs: | |
- Run-Unit-tests-Airflow-2-8 | |
- Run-Integration-tests-Airflow-2-8 | |
- Run-load-file-Integration-Airflow-2-8 | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Cache | |
uses: actions/cache@v3 | |
with: | |
path: .test_durations | |
key: test_durations-0-${{ github.run_id }} | |
- name: Collect | |
uses: actions/download-artifact@v3 | |
with: | |
path: ./python-sdk/artifacts | |
- name: Consolidate and print durations | |
run: | | |
jq -n -S \ | |
'reduce (inputs | to_entries[]) as {$key, $value} ({}; .[$key] += $value)' \ | |
artifacts/pytest_durations_*/* > .test_durations_print && \ | |
cat .test_durations_print | |
- name: Consolidate | |
run: | | |
jq -n -S \ | |
'reduce (inputs | to_entries[]) as {$key, $value} ({}; .[$key] += $value)' \ | |
artifacts/pytest_durations_*/* > .test_durations | |
build-n-publish: | |
if: github.event_name == 'release' | |
name: Build and publish Python 🐍 distributions 📦 to PyPI | |
needs: | |
- Run-Unit-tests-Airflow-2-8 | |
- Run-Integration-tests-Airflow-2-8 | |
- Run-load-file-Integration-Airflow-2-8 | |
- Run-example-dag-Integration-Airflow-2-8 | |
- Run-Optional-Packages-tests-python-sdk | |
- Generate-Constraints | |
runs-on: ubuntu-latest | |
steps: | |
- uses: actions/checkout@v2 | |
- uses: actions/setup-python@v2 | |
with: | |
python-version: '3.10' | |
architecture: 'x64' | |
- uses: actions/cache@v2 | |
with: | |
path: ~/.cache/pip | |
key: ${{ hashFiles('python-sdk/pyproject.toml') }} | |
- run: | | |
sudo apt update | |
sudo apt-get install pkg-config libxml2-dev libxmlsec1-dev libxmlsec1-openssl | |
- run: pip3 install nox | |
- run: nox -s build | |
- run: nox -s release -- dist/* | |
env: | |
TWINE_USERNAME: __token__ | |
TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} |