set log level + upload artifact for debugging #43
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Run data quality checks | |
# TODO temporary removed triggers for data quality pipeline, since it is failing for a while | |
on: | |
# workflow_dispatch: | |
# workflow_run: | |
# workflows: | |
# - Run dbt | |
# types: | |
# - completed | |
env: | |
DUCKDB_PATH: github_contributions.duckdb | |
PYTHON_VERSION: 3.9 | |
jobs: | |
soda_scan: | |
runs-on: ubuntu-latest | |
name: Run Soda Scan | |
steps: | |
- uses: actions/checkout@v2 | |
- name: Download duckdb | |
uses: dawidd6/action-download-artifact@v2 | |
with: | |
name: ${{ env.DUCKDB_PATH }} | |
workflow: dbt.yml | |
workflow_conclusion: success | |
- name: Setup Python | |
uses: actions/setup-python@v2 | |
with: | |
python-version: ${{ env.PYTHON_VERSION }} | |
- name: Install soda-core-duckdb | |
run: | | |
python -m pip install --upgrade pip | |
python -m pip install soda-core-duckdb | |
- name: Perform Soda Scan | |
run: soda scan --configuration soda_configuration.yml soda_checks.yml -d github_contributions | |
env: | |
SODA_CLOUD_API_KEY: ${{ secrets.SODA_CLOUD_API_KEY }} | |
SODA_CLOUD_API_SECRET: ${{ secrets.SODA_CLOUD_API_SECRET }} | |
SCHEMA: main |