Skip to content

dbt-core@main with dbt-common@main #1513

dbt-core@main with dbt-common@main

dbt-core@main with dbt-common@main #1513

# **what?**
# Runs all tests in dbt-core with this branch of dbt-common to ensure nothing is broken
# **why?**
# Ensure dbt-common changes do not break dbt-core
# **when?**
# This will run when trying to merge a PR into main.
# It can also be manually triggered.
# This workflow can be skipped by adding the "Skip Core Testing" label to the PR. This is
# useful when making a change in both `dbt-core` and `dbt-common` where the changes are dependant
# and cause the other repository to break.
name: "dbt-core Tests"
run-name: >-
${{ (github.event_name == 'workflow_dispatch' || github.event_name == 'workflow_call')
&& format('dbt-core@{0} with dbt-common@{1}', inputs.dbt-core-ref, inputs.dbt-common-ref)
|| 'dbt-core@main with dbt-common branch' }}
on:
merge_group:
types: [checks_requested]
pull_request:
workflow_dispatch:
inputs:
dbt-core-ref:
description: "The branch of dbt-core to test against"
default: "main"
dbt-common-ref:
description: "The branch of dbt-common to test against"
default: "main"
workflow_call:
inputs:
dbt-core-ref:
description: "The branch of dbt-core to test against"
type: string
required: true
default: "main"
dbt-common-ref:
description: "The branch of dbt-common to test against"
type: string
required: true
default: "main"
permissions: read-all
# will cancel previous workflows triggered by the same event
# and for the same ref for PRs/merges or same SHA otherwise
# and for the same inputs on workflow_dispatch or workflow_call
concurrency:
group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(fromJson('["pull_request", "merge_group"]'), github.event_name) && github.event.pull_request.head.ref || github.sha }}-${{ contains(fromJson('["workflow_call", "workflow_dispatch"]'), github.event_name) && github.event.inputs.dbt-core-ref && github.event.inputs.dbt-common-ref || github.sha }}
cancel-in-progress: true
defaults:
run:
shell: bash
# top-level adjustments can be made here
env:
# number of parallel processes to spawn for python integration testing
PYTHON_INTEGRATION_TEST_WORKERS: 5
jobs:
job-prep:
# This allow us to run the workflow on pull_requests as well so we can always run unit tests
# and only run integration tests on merge for time purposes
name: Setup Repo Refs
runs-on: ubuntu-latest
outputs:
dbt-core-ref: ${{ steps.core-ref.outputs.ref }}
dbt-common-ref: ${{ steps.common-ref.outputs.ref }}
steps:
- name: "Input Refs"
id: job-inputs
run: |
echo "inputs.dbt-core-ref=${{ inputs.dbt-core-ref }}"
echo "inputs.dbt-common-ref=${{ inputs.dbt-common-ref }}"
- name: "Determine dbt-core ref"
id: core-ref
run: |
if [[ -z "${{ inputs.dbt-core-ref }}" ]]; then
REF="main"
else
REF=${{ inputs.dbt-core-ref }}
fi
echo "ref=$REF" >> $GITHUB_OUTPUT
- name: "Determine dbt-common ref"
id: common-ref
run: |
if [[ -z "${{ inputs.dbt-common-ref }}" ]]; then
# these will be commits instead of branches
if [[ "${{ github.event_name }}" == "merge_group" ]]; then
REF=${{ github.event.merge_group.head_sha }}
else
REF=${{ github.event.pull_request.base.sha }}
fi
else
REF=${{ inputs.dbt-common-ref }}
fi
echo "ref=$REF" >> $GITHUB_OUTPUT
- name: "Final Refs"
run: |
echo "dbt-core-ref=${{ steps.core-ref.outputs.ref }}"
echo "dbt-common-ref=${{ steps.common-ref.outputs.ref }}"
dbt-core-unit-test:
name: "dbt-core unit tests"
needs: [job-prep]
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- name: "Check out dbt-core@${{ needs.job-prep.outputs.dbt-core-ref }}"
uses: actions/checkout@v4
with:
repository: dbt-labs/dbt-core
ref: ${{ needs.job-prep.outputs.dbt-core-ref }}
- name: "Set up Python 3.11"
uses: actions/setup-python@v5
with:
python-version: "3.11"
- name: "Upgrade pip, install tox"
run: |
python -m pip install --upgrade pip
python -m pip --version
python -m pip install tox
tox --version
- name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}"
run: |
./scripts/update_dev_packages.sh ${{ github.repository.name}} ${{ needs.job-prep.outputs.dbt-common-ref }}
- name: "Run unit tests"
# Doing the check here instead of the top level because this is job a required check, the
# label just means we should skip the tests
if: ${{ !contains(github.event.label.name, 'Skip Core Testing')}}
run: tox
env:
TOXENV: unit
- name: "Check installed versions"
run: pip freeze
dbt-core-integration-metadata:
name: "integration test metadata generation"
runs-on: ubuntu-latest
if: ${{ github.event_name != 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}}
outputs:
split-groups: ${{ steps.generate-split-groups.outputs.split-groups }}
include: ${{ steps.generate-include.outputs.include }}
steps:
- name: "generate split-groups"
id: generate-split-groups
run: |
MATRIX_JSON="["
for B in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
MATRIX_JSON+=$(sed 's/^/"/;s/$/"/' <<< "${B}")
done
MATRIX_JSON="${MATRIX_JSON//\"\"/\", \"}"
MATRIX_JSON+="]"
echo "split-groups=${MATRIX_JSON}"
echo "split-groups=${MATRIX_JSON}" >> $GITHUB_OUTPUT
- name: "generate include"
id: generate-include
run: |
INCLUDE=('"python-version":"3.9","os":"windows-latest"' '"python-version":"3.9","os":"macos-14"' )
INCLUDE_GROUPS="["
for include in ${INCLUDE[@]}; do
for group in $(seq 1 ${{ env.PYTHON_INTEGRATION_TEST_WORKERS }}); do
INCLUDE_GROUPS+=$(sed 's/$/, /' <<< "{\"split-group\":\"${group}\",${include}}")
done
done
INCLUDE_GROUPS=$(echo $INCLUDE_GROUPS | sed 's/,*$//g')
INCLUDE_GROUPS+="]"
echo "include=${INCLUDE_GROUPS}"
echo "include=${INCLUDE_GROUPS}" >> $GITHUB_OUTPUT
dbt-core-integration-tests:
name: "(${{ matrix.split-group }}) integration test / python ${{ matrix.python-version }} / ${{ matrix.os }}"
if: ${{ github.event_name != 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}}
runs-on: ${{ matrix.os }}
timeout-minutes: 30
needs: [job-prep, dbt-core-integration-metadata]
strategy:
fail-fast: false
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
os: [ubuntu-20.04]
split-group: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.split-groups) }}
include: ${{ fromJson(needs.dbt-core-integration-metadata.outputs.include) }}
env:
DBT_INVOCATION_ENV: github-actions
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
DBT_TEST_USER_3: dbt_test_user_3
steps:
- name: "Check out the repository@${{ needs.job-prep.outputs.dbt-core-ref }}"
uses: actions/checkout@v4
with:
repository: dbt-labs/dbt-core
ref: ${{ needs.job-prep.outputs.dbt-core-ref }}
- name: "Set up Python ${{ matrix.python-version }}"
uses: actions/setup-python@v5
with:
python-version: ${{ matrix.python-version }}
- name: "Set up postgres (linux)"
if: runner.os == 'Linux'
uses: ./.github/actions/setup-postgres-linux
- name: "Set up postgres (macos)"
if: runner.os == 'macOS'
uses: ./.github/actions/setup-postgres-macos
- name: "Set up postgres (windows)"
if: runner.os == 'Windows'
uses: ./.github/actions/setup-postgres-windows
- name: "Upgrade pip, install tox"
run: |
python -m pip install --upgrade pip
python -m pip --version
python -m pip install tox
tox --version
- name: "Update dbt-common dependency to ${{ needs.job-prep.outputs.dbt-common-ref }}"
run: |
./scripts/update_dev_packages.sh ${{ github.repository.name}} ${{ needs.job-prep.outputs.dbt-common-ref }}
- name: "Run Functional tests"
run: tox -- --ddtrace
env:
TOXENV: integration
DBT_INVOCATION_ENV: github-actions
DBT_TEST_USER_1: dbt_test_user_1
DBT_TEST_USER_2: dbt_test_user_2
DBT_TEST_USER_3: dbt_test_user_3
DD_CIVISIBILITY_AGENTLESS_ENABLED: true
DD_API_KEY: ${{ secrets.DATADOG_API_KEY }}
DD_SITE: datadoghq.com
DD_ENV: ci
DD_SERVICE: dbt-core
PYTEST_ADDOPTS: ${{ format('--splits {0} --group {1}', env.PYTHON_INTEGRATION_TEST_WORKERS, matrix.split-group) }}
- name: "Check installed versions"
run: pip freeze
integration-report:
if: ${{ always() }}
name: "dbt-core Integration Test Suite Report"
runs-on: ubuntu-latest
needs: [dbt-core-integration-tests]
steps:
- name: "Integration Tests Failed"
if: ${{ contains(needs.dbt-core-integration-tests.result, 'failure') || contains(needs.dbt-core-integration-tests.result, 'cancelled') }}
# when this is true the next step won't execute
run: |
echo "::notice title='Integration test suite failed'"
exit 1
- name: "Integration Tests Passed"
if: ${{ github.event_name != 'pull_request' }}
run: |
echo "::notice title='Integration test suite passed'"
- name: "Integration Tests Skipped on Pull Request"
if: ${{ github.event_name == 'pull_request' && !contains(github.event.label.name, 'Skip Core Testing')}}
run: |
echo "::notice title='Integration test suite skipped on Pull Requests - they will run on merge'"
- name: "Integration Tests Skipped by Label"
if: ${{ contains(github.event.label.name, 'Skip Core Testing')}}
run: |
echo "::notice title='dbt-core test suite skipped due to Skip Core Testing label'"