From fbb1a88b27eda6be28d4ecf5cb262a49f3f202be Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 15:42:48 -0500 Subject: [PATCH 01/63] create new batch of workflows --- .github/workflows/_code-quality.yml | 39 +++++++++ .github/workflows/_publish-internal.yml | 102 ++++++++++++++++++++++ .github/workflows/_publish-pypi.yml | 85 ++++++++++++++++++ .github/workflows/_unit-tests.yml | 66 ++++++++++++++ .github/workflows/publish.yml | 58 ++++++++++++ .github/workflows/pull-request-checks.yml | 38 ++++++++ 6 files changed, 388 insertions(+) create mode 100644 .github/workflows/_code-quality.yml create mode 100644 .github/workflows/_publish-internal.yml create mode 100644 .github/workflows/_publish-pypi.yml create mode 100644 .github/workflows/_unit-tests.yml create mode 100644 .github/workflows/publish.yml create mode 100644 .github/workflows/pull-request-checks.yml diff --git a/.github/workflows/_code-quality.yml b/.github/workflows/_code-quality.yml new file mode 100644 index 00000000..05d2e6da --- /dev/null +++ b/.github/workflows/_code-quality.yml @@ -0,0 +1,39 @@ +name: "Code quality" + +on: + workflow_call: + inputs: + branch: + description: "Choose the branch to check" + type: string + default: "main" + repository: + description: "Choose the repository to check, when using a fork" + type: string + default: "dbt-labs/dbt-adapters" + workflow_dispatch: + inputs: + branch: + description: "Choose the branch to check" + type: string + default: "main" + repository: + description: "Choose the repository to check, when using a fork" + type: string + default: "dbt-labs/dbt-adapters" + +permissions: + contents: read + +jobs: + code-quality: + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + repository: ${{ inputs.repository }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml new file mode 100644 index 00000000..815c0f70 --- /dev/null +++ b/.github/workflows/_publish-internal.yml @@ -0,0 +1,102 @@ +name: "Publish internally" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to publish" + type: string + default: "dbt-adapters" + deploy-to: + description: "Choose whether to publish to test or prod" + type: string + default: "prod" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + workflow_dispatch: + inputs: + package: + description: "Choose the package to publish" + type: choice + options: ["dbt-adapters"] + deploy-to: + description: "Choose whether to publish to test or prod" + type: environment + default: "test" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + +defaults: + run: + shell: bash + +jobs: + publish: + runs-on: ${{ vars.DEFAULT_RUNNER }} + environment: + name: ${{ inputs.deploy-to }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - uses: pypa/hatch@install + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - id: package + run: | + # strip the pre-release off to find all iterations of this patch + hatch version release + echo "version=$(hatch version)" >> $GITHUB_OUTPUT + working-directory: ./${{ inputs.package }} + - id: published + run: | + versions_published="$(aws codeartifact list-package-versions \ + --domain ${{ vars.AWS_DOMAIN }} \ + --repository ${{ vars.AWS_REPOSITORY }} \ + --format pypi \ + --package ${{ inputs.package }} \ + --output json \ + --query 'versions[*].version' | jq -r '.[]' | grep "^${{ steps.package.outputs.version }}" || true )" # suppress pipefail only here + echo "versions=$(echo "${versions_published[*]}"| tr '\n' ',')" >> $GITHUB_OUTPUT + - id: next + uses: dbt-labs/dbt-release/.github/actions/next-cloud-release-version@main + with: + version_number: ${{ steps.package.outputs.version }} + versions_published: ${{ steps.published.outputs.versions }} + - name: "Update version to internal PyPI format" + run: | + VERSION=${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD) + tee <<< "version = \"$VERSION\"" ./src/dbt/adapters/athena/__version__.py + working-directory: ./${{ inputs.package }} + - name: "Remove dbt-core from build requirements" + run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml + working-directory: ./${{ inputs.package }} + - run: | + export HATCH_INDEX_USER=${{ secrets.AWS_USER }} + + export HATCH_INDEX_AUTH=$(aws codeartifact get-authorization-token \ + --domain ${{ vars.AWS_DOMAIN }} \ + --output text \ + --query authorizationToken) + + export HATCH_INDEX_REPO=$(aws codeartifact get-repository-endpoint \ + --domain ${{ vars.AWS_DOMAIN }} \ + --repository ${{ vars.AWS_REPOSITORY }} \ + --format pypi \ + --output text \ + --query repositoryEndpoint) + + hatch build --clean + hatch run build:check-all + hatch publish + working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml new file mode 100644 index 00000000..d8eed4c1 --- /dev/null +++ b/.github/workflows/_publish-pypi.yml @@ -0,0 +1,85 @@ +name: "Publish to PyPI" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to publish" + type: string + default: "dbt-adapters" + deploy-to: + description: "Choose whether to publish to test or prod" + type: string + default: "prod" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + workflow_dispatch: + inputs: + package: + description: "Choose the package to publish" + type: choice + options: ["dbt-adapters"] + deploy-to: + description: "Choose whether to publish to test or prod" + type: environment + default: "test" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + publish: + runs-on: ${{ vars.DEFAULT_RUNNER }} + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }} + permissions: + # this permission is required for trusted publishing + # see https://github.com/marketplace/actions/pypi-publish + id-token: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - uses: pypa/hatch@install + # hatch will build using test PyPI first and fall back to prod PyPI when deploying to test + # this is done via environment variables in the test environment in GitHub + - run: hatch build && hatch run build:check-all + working-directory: ./${{ inputs.package }} + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + packages-dir: ${{ inputs.package }}/dist/ + + verify: + runs-on: ${{ vars.DEFAULT_RUNNER }} + needs: publish + # check the correct index + environment: + name: ${{ inputs.deploy-to }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - id: version + run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT + working-directory: ./${{ inputs.package }} + - uses: nick-fields/retry@v3 + with: + timeout_seconds: 10 + retry_wait_seconds: 10 + max_attempts: 15 # 5 minutes: (10s timeout + 10s delay) * 15 attempts + command: wget ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }}/${{ steps.version.outputs.version }} diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml new file mode 100644 index 00000000..be2cac09 --- /dev/null +++ b/.github/workflows/_unit-tests.yml @@ -0,0 +1,66 @@ +name: "Unit tests" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to test" + type: string + default: "dbt-adapters" + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-adapters" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: string + default: ${{ vars.DEFAULT_PYTHON_VERSION }} + workflow_dispatch: + inputs: + package: + description: "Choose the package to test" + type: choice + options: ["dbt-adapters"] + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-adapters" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: choice + options: ["3.9", "3.10", "3.11", "3.12"] + +permissions: + contents: read + +jobs: + unit-tests: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + repository: ${{ inputs.repository }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - uses: pypa/hatch@install + - run: hatch run unit-tests + shell: bash + working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..31558056 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,58 @@ +name: "Publish" + +on: + workflow_dispatch: + inputs: + package: + description: "Choose the package to publish" + type: choice + options: + - "dbt-adapters" + - "dbt-tests-adapter" + deploy-to: + description: "Choose whether to publish to test or prod" + type: environment + default: "prod" + branch: + description: "Choose the branch to publish from" + type: string + default: "main" + pypi-internal: + description: "Publish Internally" + type: boolean + default: true + pypi-public: + description: "Publish to PyPI" + type: boolean + default: false + +# don't attempt to release the same target in parallel +concurrency: + group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.deploy-to }} + cancel-in-progress: true + +jobs: + unit-tests: + uses: ./.github/workflows/_unit-tests.yml + with: + package: ${{ inputs.package }} + branch: ${{ inputs.branch }} + + publish-internal: + if: ${{ inputs.pypi-internal == true }} + needs: unit-tests + uses: ./.github/workflows/_publish-internal.yml + with: + package: ${{ inputs.package }} + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} + secrets: inherit + + publish-pypi: + if: ${{ inputs.pypi-public == true }} + needs: unit-tests + uses: ./.github/workflows/_publish-pypi.yml + with: + package: ${{ inputs.package }} + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml new file mode 100644 index 00000000..2322ff91 --- /dev/null +++ b/.github/workflows/pull-request-checks.yml @@ -0,0 +1,38 @@ +name: "Pull request checks" + +on: + pull_request_target: + types: [opened, reopened, synchronize] + +# only run this once per PR at a time +concurrency: + group: ${{ github.workflow }}-${{ github.event.number }} + cancel-in-progress: true + +jobs: + code-quality: + uses: ./.github/workflows/_code-quality.yml + with: + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + unit-tests: + uses: ./.github/workflows/_unit-tests.yml + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + strategy: + matrix: + package: ["dbt-adapters"] + + # This job does nothing and is only used for branch protection + results: + name: "Pull request checks" + if: always() + needs: [code-quality, unit-tests] + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + - uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} From 378fe73c694b74a6cacc2f9fa960d4c592debc7a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 17:45:30 -0500 Subject: [PATCH 02/63] remove old workflows --- .github/workflows/build.yml | 54 -- .github/workflows/changelog-existence.yml | 37 -- .github/workflows/code-quality.yml | 34 -- .github/workflows/docs-issue.yml | 41 -- .github/workflows/github-release.yml | 259 ---------- .github/workflows/precommit-autoupdate.yml | 22 - .github/workflows/release.yml | 180 ------- .github/workflows/release_prep_hatch.yml | 542 --------------------- .github/workflows/resubmit-for-triage.yml | 31 -- .github/workflows/stale.yml | 30 -- .github/workflows/unit-tests.yml | 49 -- 11 files changed, 1279 deletions(-) delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/changelog-existence.yml delete mode 100644 .github/workflows/code-quality.yml delete mode 100644 .github/workflows/docs-issue.yml delete mode 100644 .github/workflows/github-release.yml delete mode 100644 .github/workflows/precommit-autoupdate.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/release_prep_hatch.yml delete mode 100644 .github/workflows/resubmit-for-triage.yml delete mode 100644 .github/workflows/stale.yml delete mode 100644 .github/workflows/unit-tests.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 00afd704..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,54 +0,0 @@ -# **what?** -# Verifies python build on all code commited to the repository. This workflow -# should not require any secrets since it runs for PRs from forked repos. By -# default, secrets are not passed to workflows running from a forked repos. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to main, and when manually triggered. - -name: "Build" - -on: - push: - branches: - - "main" - pull_request: - merge_group: - types: [checks_requested] - workflow_dispatch: - workflow_call: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - build: - name: Build, Test and publish to PyPi - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/changelog-existence.yml deleted file mode 100644 index 8732177f..00000000 --- a/.github/workflows/changelog-existence.yml +++ /dev/null @@ -1,37 +0,0 @@ -# **what?** -# Checks that a file has been committed under the /.changes directory -# as a new CHANGELOG entry. Cannot check for a specific filename as -# it is dynamically generated by change type and timestamp. -# This workflow runs on pull_request_target because it requires -# secrets to post comments. - -# **why?** -# Ensure code change gets reflected in the CHANGELOG. - -# **when?** -# This will run for all PRs going into main. It will -# run when they are opened, reopened, when any label is added or removed -# and when new code is pushed to the branch. The action will get -# skipped if the 'Skip Changelog' label is present is any of the labels. - -name: Check Changelog Entry - -on: - pull_request_target: - types: [opened, reopened, labeled, unlabeled, synchronize] - -defaults: - run: - shell: bash - -permissions: - contents: read - pull-requests: write - -jobs: - changelog: - uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main - with: - changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' - skip_label: 'Skip Changelog' - secrets: inherit diff --git a/.github/workflows/code-quality.yml b/.github/workflows/code-quality.yml deleted file mode 100644 index 9c203847..00000000 --- a/.github/workflows/code-quality.yml +++ /dev/null @@ -1,34 +0,0 @@ -name: Code Quality - -on: - push: - branches: - - "main" - - "*.latest" - pull_request: - workflow_dispatch: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -jobs: - code-quality: - name: Code Quality - runs-on: ubuntu-latest - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Run code quality - shell: bash - run: hatch run code-quality diff --git a/.github/workflows/docs-issue.yml b/.github/workflows/docs-issue.yml deleted file mode 100644 index f49cf517..00000000 --- a/.github/workflows/docs-issue.yml +++ /dev/null @@ -1,41 +0,0 @@ -# **what?** -# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed - -# **why?** -# To reduce barriers for keeping docs up to date - -# **when?** -# When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. - - -name: Open issues in docs.getdbt.com repo when an issue is labeled -run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" - -on: - issues: - types: [labeled, closed] - -defaults: - run: - shell: bash - -permissions: - issues: write # comments on issues - -jobs: - open_issues: - # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. - # If this logic does not exist in this workflow, it runs the - # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having - # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we - # decide if it should run or not. - if: | - (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( - (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || - (github.event.action == 'labeled' && github.event.label.name == 'user docs')) - uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main - with: - issue_repository: "dbt-labs/docs.getdbt.com" - issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" - issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." - secrets: inherit diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml deleted file mode 100644 index ad0cc2d8..00000000 --- a/.github/workflows/github-release.yml +++ /dev/null @@ -1,259 +0,0 @@ -# **what?** -# Create a new release on GitHub and include any artifacts in the `/dist` directory of the GitHub artifacts store. -# -# Inputs: -# sha: The commit to attach to this release -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# changelog_path: Path to the changelog file for release notes -# test_run: Test run (Publish release as draft) -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call after a successful build. Build artifacts should be ready to release and live in a dist/ directory. -# -# This workflow expects the artifacts to already be built and living in the artifact store of the workflow. -# -# Validation Checks -# -# 1. If no release already exists for this commit and version, create the tag and release it to GitHub. -# 2. If a release already exists for this commit, skip creating the release but finish with a success. -# 3. If a release exists for this commit under a different tag, fail. -# 4. If the commit is already associated with a different release, fail. - -name: GitHub Release - -on: - workflow_call: - inputs: - sha: - description: The commit to attach to this release - required: true - type: string - version_number: - description: The release version number (i.e. 1.0.0b1) - required: true - type: string - changelog_path: - description: Path to the changelog file for release notes - required: true - type: string - test_run: - description: Test run (Publish release as draft) - required: true - type: boolean - archive_name: - description: artifact name to download - required: true - type: string - outputs: - tag: - description: The path to the changelog for this version - value: ${{ jobs.check-release-exists.outputs.tag }} - -permissions: - contents: write - -env: - REPO_LINK: ${{ github.server_url }}/${{ github.repository }} - NOTIFICATION_PREFIX: "[GitHub Release]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The release version number: ${{ inputs.version_number }} - echo Expected Changelog path: ${{ inputs.changelog_path }} - echo Test run: ${{ inputs.test_run }} - echo Repo link: ${{ env.REPO_LINK }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - - check-release-exists: - runs-on: ubuntu-latest - outputs: - exists: ${{ steps.release_check.outputs.exists }} - draft_exists: ${{ steps.release_check.outputs.draft_exists }} - tag: ${{ steps.set_tag.outputs.tag }} - - steps: - - name: "Generate Release Tag" - id: set_tag - run: echo "tag=v${{ inputs.version_number }}" >> $GITHUB_OUTPUT - - # When the GitHub CLI doesn't find a release for the given tag, it will exit 1 with a - # message of "release not found". In our case, it's not an actual error, just a - # confirmation that the release does not already exists so we can go ahead and create it. - # The `|| true` makes it so the step does not exit with a non-zero exit code - # Also check if the release already exists is draft state. If it does, and we are not - # testing then we can publish that draft as is. If it's in draft and we are testing, skip the - # release. - - name: "Check If Release Exists For Tag ${{ steps.set_tag.outputs.tag }}" - id: release_check - run: | - output=$((gh release view ${{ steps.set_tag.outputs.tag }} --json isDraft,targetCommitish --repo ${{ env.REPO_LINK }}) 2>&1) || true - if [[ "$output" == "release not found" ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} does not exist." - message="Check passed." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - commit=$(jq -r '.targetCommitish' <<< "$output") - if [[ $commit != ${{ inputs.sha }} ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists for commit $commit!" - message="Cannot create a new release for commit ${{ inputs.sha }}. Exiting." - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - isDraft=$(jq -r '.isDraft' <<< "$output") - if [[ $isDraft == true ]] && [[ ${{ inputs.test_run }} == false ]] - then - title="Release tag ${{ steps.set_tag.outputs.tag }} already associated with the draft release." - message="Release workflow will publish the associated release." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=true" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists." - message="Skip GitHub Release Publishing." - echo "exists=true" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ env.REPO_LINK }} - - - name: "[DEBUG] Log Job Outputs" - run: | - echo exists: ${{ steps.release_check.outputs.exists }} - echo draft_exists: ${{ steps.release_check.outputs.draft_exists }} - echo tag: ${{ steps.set_tag.outputs.tag }} - - skip-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'true' - - steps: - - name: "Tag Exists, Skip GitHub Release Job" - run: | - echo title="A tag already exists for ${{ needs.check-release-exists.outputs.tag }} and commit." - echo message="Skipping GitHub release." - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - audit-release-different-commit: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'false' - - steps: - - name: "Check If Release Already Exists For Commit" - uses: cardinalby/git-get-release-action@1.2.4 - id: check_release_commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - commitSha: ${{ inputs.sha }} - doNotFailIfNotFound: true # returns blank outputs when not found instead of error - searchLimit: 15 # Since we only care about recent releases, speed up the process - - - name: "[DEBUG] Print Release Details" - run: | - echo steps.check_release_commit.outputs.id: ${{ steps.check_release_commit.outputs.id }} - echo steps.check_release_commit.outputs.tag_name: ${{ steps.check_release_commit.outputs.tag_name }} - echo steps.check_release_commit.outputs.target_commitish: ${{ steps.check_release_commit.outputs.target_commitish }} - echo steps.check_release_commit.outputs.prerelease: ${{ steps.check_release_commit.outputs.prerelease }} - - # Since we already know a release for this tag does not exist, if we find anything it's for the wrong tag, exit - - name: "Check If The Tag Matches The Version Number" - if: steps.check_release_commit.outputs.id != '' - run: | - title="Tag ${{ steps.check_release_commit.outputs.tag_name }} already exists for this commit!" - message="Cannot create a new tag for ${{ needs.check-release-exists.outputs.tag }} for the same commit" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - - publish-draft-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: >- - needs.check-release-exists.outputs.draft_exists == 'true' && - inputs.test_run == false - - steps: - - name: "Publish Draft Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release edit $TAG --draft=false --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - - create-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: needs.check-release-exists.outputs.draft_exists == 'false' - - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - - name: "Download Artifact ${{ inputs.archive_name }}" - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.archive_name }} - path: dist/ - - - name: "[DEBUG] Display Structure Of Expected Files" - run: | - ls -R .changes - ls -l dist - - - name: "Set Release Type" - id: release_type - run: | - if ${{ contains(inputs.version_number, 'rc') || contains(inputs.version_number, 'b') }} - then - echo Release will be set as pre-release - echo "prerelease=--prerelease" >> $GITHUB_OUTPUT - else - echo This is not a prerelease - fi - - - name: "Set As Draft Release" - id: draft - run: | - if [[ ${{ inputs.test_run }} == true ]] - then - echo Release will be published as draft - echo "draft=--draft" >> $GITHUB_OUTPUT - else - echo This is not a draft release - fi - - - name: "GitHub Release Workflow Annotation" - run: | - title="Release ${{ needs.check-release-exists.outputs.tag }}" - message="Configuration: ${{ steps.release_type.outputs.prerelease }} ${{ steps.draft.outputs.draft }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - - name: "Create New GitHub Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release create $TAG ./dist/* --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE $DRAFT --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - TITLE: ${{ github.event.repository.name }} ${{ needs.check-release-exists.outputs.tag }} - RELEASE_NOTES: ${{ inputs.changelog_path }} - COMMIT: ${{ inputs.sha }} - PRERELEASE: ${{ steps.release_type.outputs.prerelease }} - DRAFT: ${{ steps.draft.outputs.draft }} diff --git a/.github/workflows/precommit-autoupdate.yml b/.github/workflows/precommit-autoupdate.yml deleted file mode 100644 index 74976c48..00000000 --- a/.github/workflows/precommit-autoupdate.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Run pre-commit autoupdate" - -on: - schedule: - - cron: "30 1 * * SAT" - workflow_dispatch: - -permissions: - contents: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }} - cancel-in-progress: true - -jobs: - precommit-autoupdate: - name: "Run pre-commit autoupdate" - uses: dbt-labs/actions/.github/workflows/pre-commit-autoupdate.yml - secrets: - TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} - SLACK_WEBHOOK_PR_URL: ${{ secrets.SLACK_DEV_ADAPTER_PULL_REQUESTS }} - SLACK_WEBHOOK_ALERTS_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 828350dd..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,180 +0,0 @@ -name: Release -run-name: Release ${{ inputs.package }}==${{ inputs.version_number }} to ${{ inputs.deploy-to }} - -on: - workflow_dispatch: - inputs: - package: - type: choice - description: Choose what to publish - options: - - dbt-adapters - - dbt-tests-adapter - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - deploy-to: - type: choice - description: Choose where to publish - options: - - prod - - test - default: prod - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - - workflow_call: - inputs: - package: - type: string - description: Choose what to publish - required: true - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - deploy-to: - type: string - default: prod - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - -# this is the permission that allows creating a new release -permissions: - contents: write - id-token: write - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}-${{ inputs.package }}-${{ inputs.deploy-to }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - release-inputs: - name: "Release inputs" - runs-on: ubuntu-latest - outputs: - working-dir: ${{ steps.release-inputs.outputs.working-dir }} - run-unit-tests: ${{ steps.release-inputs.outputs.run-unit-tests }} - archive-name: ${{ steps.release-inputs.outputs.archive-name }} - steps: - - name: "Inputs" - id: release-inputs - run: | - working_dir="./" - run_unit_tests=true - archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-to }} - - if test "${{ inputs.package }}" = "dbt-tests-adapter" - then - working_dir="./dbt-tests-adapter/" - run_unit_tests=false - fi - - echo "working-dir=$working_dir" >> $GITHUB_OUTPUT - echo "run-unit-tests=$run_unit_tests" >> $GITHUB_OUTPUT - echo "archive-name=$archive_name" >> $GITHUB_OUTPUT - - - name: "[DEBUG]" - run: | - echo package : ${{ inputs.package }} - echo working-dir : ${{ steps.release-inputs.outputs.working-dir }} - echo run-unit-tests : ${{ steps.release-inputs.outputs.run-unit-tests }} - echo archive-name : ${{ steps.release-inputs.outputs.archive-name }} - - bump-version-generate-changelog: - name: "Bump package version, Generate changelog" - uses: dbt-labs/dbt-adapters/.github/workflows/release_prep_hatch.yml@main - needs: [release-inputs] - with: - version_number: ${{ inputs.version_number }} - deploy_to: ${{ inputs.deploy-to }} - nightly_release: ${{ inputs.nightly_release }} - target_branch: ${{ inputs.target_branch }} - working-dir: ${{ needs.release-inputs.outputs.working-dir }} - run-unit-tests: ${{ fromJSON(needs.release-inputs.outputs.run-unit-tests) }} - secrets: inherit - - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-and-test: - name: "Build and Test" - needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main - - - name: "Build ${{ inputs.package }}" - uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main - with: - working-dir: ${{ needs.release-inputs.outputs.working-dir }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} - - github-release: - name: "GitHub Release" - # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapters assets to the same release - if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} - needs: [release-inputs, build-and-test, bump-version-generate-changelog] - uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.deploy-to == 'test' && true || false }} - archive_name: ${{ needs.release-inputs.outputs.archive-name }} - - pypi-release: - name: "Publish to PyPI" - runs-on: ubuntu-latest - needs: [release-inputs, build-and-test] - environment: - name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_PROJECT_URL }} - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - - - name: "Publish to PyPI" - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main - with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml deleted file mode 100644 index a6105786..00000000 --- a/.github/workflows/release_prep_hatch.yml +++ /dev/null @@ -1,542 +0,0 @@ -# **what?** -# Perform the version bump, generate the changelog and run tests. -# -# Inputs: -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# target_branch: The branch that we will release from -# env_setup_script_path: Path to the environment setup script -# deploy_to: If we are deploying to prod or test, if test then release from branch -# nightly_release: Identifier that this is nightly release -# -# Outputs: -# final_sha: The sha that will actually be released. This can differ from the -# input sha if adding a version bump and/or changelog -# changelog_path: Path to the changelog file (ex .changes/1.2.3-rc1.md) -# -# Branching strategy: -# - During execution workflow execution the temp branch will be generated. -# - For normal runs the temp branch will be removed once changes were merged to target branch; -# - For test runs we will keep temp branch and will use it for release; -# Naming strategy: -# - For normal runs: prep-release/${{ inputs.deploy_to}}/${{ inputs.version_number }}_$GITHUB_RUN_ID -# - For nightly releases: prep-release/nightly-release/${{ inputs.version_number }}_$GITHUB_RUN_ID -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call when ready to kick off a build and release -# -# Validation Checks -# -# 1. Bump the version if it has not been bumped -# 2. Generate the changelog (via changie) if there is no markdown file for this version -# - -name: Version Bump and Changelog Generation -run-name: Bump to ${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog -on: - workflow_call: - inputs: - version_number: - required: true - type: string - deploy_to: - type: string - default: prod - required: false - nightly_release: - type: boolean - default: false - required: false - env_setup_script_path: - type: string - required: false - default: '' - run-unit-tests: - type: boolean - default: false - run-integration-tests: - type: boolean - default: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - working-dir: - description: "The working directory to use for run statements" - type: string - default: "./" - outputs: - changelog_path: - description: The path to the changelog for this version - value: ${{ jobs.audit-changelog.outputs.changelog_path }} - final_sha: - description: The sha that will actually be released - value: ${{ jobs.determine-release-branch.outputs.final_sha }} - secrets: - FISHTOWN_BOT_PAT: - description: "Token to commit/merge changes into branches" - required: true - IT_TEAM_MEMBERSHIP: - description: "Token that can view org level teams" - required: true - -permissions: - contents: write - -defaults: - run: - shell: bash - -env: - PYTHON_TARGET_VERSION: 3.11 - NOTIFICATION_PREFIX: "[Release Preparation]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - - steps: - - name: "[DEBUG] Print Variables" - run: | - # WORKFLOW INPUTS - echo The release version number: ${{ inputs.version_number }} - echo Deploy to: ${{ inputs.deploy_to }} - echo Target branch: ${{ inputs.target_branch }} - echo Nightly release: ${{ inputs.nightly_release }} - echo Optional env setup script: ${{ inputs.env_setup_script_path }} - echo run-unit-tests: ${{ inputs.run-unit-tests }} - echo run-integration-tests: ${{ inputs.run-integration-tests }} - echo working-dir: ${{ inputs.working-dir }} - # ENVIRONMENT VARIABLES - echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - audit-changelog: - runs-on: ubuntu-latest - - outputs: - changelog_path: ${{ steps.set_path.outputs.changelog_path }} - exists: ${{ steps.set_existence.outputs.exists }} - base_version: ${{ steps.semver.outputs.base-version }} - prerelease: ${{ steps.semver.outputs.pre-release }} - is_prerelease: ${{ steps.semver.outputs.is-pre-release }} - - steps: - - name: "Checkout ${{ github.repository }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: "Audit Version And Parse Into Parts" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.1 - with: - version: ${{ inputs.version_number }} - - - name: "Set Changelog Path" - id: set_path - run: | - path=".changes/" - if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] - then - path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" - else - path+="${{ steps.semver.outputs.base-version }}.md" - fi - # Send notification - echo "changelog_path=$path" >> $GITHUB_OUTPUT - title="Changelog path" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" - - name: "Set Changelog Existence For Subsequent Jobs" - id: set_existence - run: | - does_exist=false - if test -f ${{ steps.set_path.outputs.changelog_path }} - then - does_exist=true - fi - echo "exists=$does_exist">> $GITHUB_OUTPUT - - name: "[Notification] Set Changelog Existence For Subsequent Jobs" - run: | - title="Changelog exists" - if [[ ${{ steps.set_existence.outputs.exists }} == true ]] - then - message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" - else - message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" - fi - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} - echo exists: ${{ steps.set_existence.outputs.exists }} - echo base_version: ${{ steps.semver.outputs.base-version }} - echo prerelease: ${{ steps.semver.outputs.pre-release }} - echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} - - audit-version-in-code: - runs-on: ubuntu-latest - - outputs: - up_to_date: ${{ steps.version-check.outputs.up_to_date }} - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ inputs.target_branch }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: "Check Current Version In Code" - id: version-check - run: | - is_updated=false - current_version=$(hatch version) - if test "$current_version" = "${{ inputs.version_number }}" - then - is_updated=true - fi - echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT - working-directory: ${{ inputs.working-dir }} - - - name: "[Notification] Check Current Version In Code" - run: | - title="Version check" - if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] - then - message="The version in the codebase is equal to the provided version" - else - message="The version in the codebase differs from the provided version" - fi - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} - - skip-generate-changelog: - runs-on: ubuntu-latest - needs: [audit-changelog] - if: needs.audit-changelog.outputs.exists == 'true' - - steps: - - name: "Changelog Exists, Skip Generating New Changelog" - run: | - # Send notification - title="Skip changelog generation" - message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - skip-version-bump: - runs-on: ubuntu-latest - needs: [audit-version-in-code] - if: needs.audit-version-in-code.outputs.up_to_date == 'true' - - steps: - - name: "Version Already Bumped" - run: | - # Send notification - title="Skip version bump" - message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - create-temp-branch: - runs-on: ubuntu-latest - needs: [audit-changelog, audit-version-in-code] - if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' - - outputs: - branch_name: ${{ steps.variables.outputs.branch_name }} - - steps: - - name: "Checkout ${{ github.repository }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: "Generate Branch Name" - id: variables - run: | - name="prep-release/" - if [[ ${{ inputs.nightly_release }} == true ]] - then - name+="nightly-release/" - else - name+="${{ inputs.deploy_to }}/" - fi - name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" - echo "branch_name=$name" >> $GITHUB_OUTPUT - - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" - run: | - git checkout -b ${{ steps.variables.outputs.branch_name }} - git push -u origin ${{ steps.variables.outputs.branch_name }} - - name: "[Notification] Temp branch created" - run: | - # Send notification - title="Temp branch generated" - message="The ${{ steps.variables.outputs.branch_name }} branch created" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo branch_name ${{ steps.variables.outputs.branch_name }} - generate-changelog-bump-version: - runs-on: ubuntu-latest - needs: [audit-changelog, audit-version-in-code, create-temp-branch] - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v4 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - name: "Add Homebrew To PATH" - run: | - echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - - name: "Install Homebrew Packages" - run: | - brew install pre-commit - brew tap miniscruff/changie https://github.com/miniscruff/changie - brew install changie - - name: "Set json File Name" - id: json_file - run: | - echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT - - name: "Get Core Team Membership" - run: | - gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} - env: - GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} - - name: "Set Core Team Membership for Changie Contributors exclusion" - id: set_team_membership - run: | - team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) - echo $team_list - team_list_single=$(echo $team_list | tr '\n' ' ') - echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV - - name: "Delete the json File" - run: | - rm ${{ steps.json_file.outputs.name }} - - name: "Generate Release Changelog" - if: needs.audit-changelog.outputs.exists == 'false' - run: | - if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] - then - changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} - elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] - then - changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases - else # releasing a final patch with no prereleases - changie batch ${{ needs.audit-changelog.outputs.base_version }} - fi - changie merge - git status - - name: "Check Changelog Created Successfully" - if: needs.audit-changelog.outputs.exists == 'false' - run: | - title="Changelog" - if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] - then - message="Changelog file created successfully" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - else - message="Changelog failed to generate" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - - name: "Bump Version To ${{ inputs.version_number }}" - if: needs.audit-version-in-code.outputs.up_to_date == 'false' - run: | - hatch version ${{ inputs.version_number }} - working-directory: ${{ inputs.working-dir }} - - name: "[Notification] Bump Version To ${{ inputs.version_number }}" - if: needs.audit-version-in-code.outputs.up_to_date == 'false' - run: | - title="Version bump" - message="Version successfully bumped in codebase to ${{ inputs.version_number }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - # TODO: can these 2 steps be done via hatch? probably. - # this step will fail on whitespace errors but also correct them - - name: "Remove Trailing Whitespace Via Pre-commit" - continue-on-error: true - run: | - pre-commit run trailing-whitespace --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - git status - # this step will fail on newline errors but also correct them - - name: "Removing Extra Newlines Via Pre-commit" - continue-on-error: true - run: | - pre-commit run end-of-file-fixer --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - git status - - name: "Commit & Push Changes" - run: | - #Data for commit - user="Github Build Bot" - email="buildbot@fishtownanalytics.com" - commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" - #Commit changes to branch - git config user.name "$user" - git config user.email "$email" - git pull - git add . - git commit -m "$commit_message" - git push - - run-unit-tests: - if: inputs.run-unit-tests == true - runs-on: ubuntu-latest - needs: [create-temp-branch, generate-changelog-bump-version] - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v4 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - name: "Setup `hatch`" - uses: ./.github/actions/setup-hatch - - name: "Run Unit Tests" - run: hatch run unit-tests - - run-integration-tests: - runs-on: ubuntu-20.04 - needs: [create-temp-branch, generate-changelog-bump-version] - if: inputs.run-integration-tests == true - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v4 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - - name: "Setup Environment Variables - ./${{ inputs.env_setup_script_path }}" - if: inputs.env_setup_script_path != '' - run: source ./${{ inputs.env_setup_script_path }} - - - name: "Setup Environment Variables - Secrets Context" - if: inputs.env_setup_script_path != '' - uses: actions/github-script@v6 - id: check-env - with: - result-encoding: string - script: | - try { - const { SECRETS_CONTEXT, INTEGRATION_TESTS_SECRETS_PREFIX } = process.env - const secrets = JSON.parse(SECRETS_CONTEXT) - if (INTEGRATION_TESTS_SECRETS_PREFIX) { - for (const [key, value] of Object.entries(secrets)) { - if (key.startsWith(INTEGRATION_TESTS_SECRETS_PREFIX)) { - core.exportVariable(key, value) - } - } - } else { - core.info("The INTEGRATION_TESTS_SECRETS_PREFIX env variable is empty or not defined, skipping the secrets setup.") - } - } catch (err) { - core.error("Error while reading or parsing the JSON") - core.setFailed(err) - } - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: "Set up Python & Hatch - ${{ env.PYTHON_TARGET_VERSION }}" - uses: ./.github/actions/setup-python-env - with: - python-version: ${{ env.PYTHON_TARGET_VERSION }} - - - name: Run tests - run: hatch run integration-tests - - merge-changes-into-target-branch: - runs-on: ubuntu-latest - needs: [run-unit-tests, run-integration-tests, create-temp-branch, audit-version-in-code, audit-changelog] - if: | - !failure() && !cancelled() && - inputs.deploy_to == 'prod' && - ( - needs.audit-changelog.outputs.exists == 'false' || - needs.audit-version-in-code.outputs.up_to_date == 'false' - ) - steps: - - name: "[Debug] Print Variables" - run: | - echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} - echo inputs.deploy_to: ${{ inputs.deploy_to }} - echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} - echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - - name: "Checkout Repo ${{ github.repository }}" - uses: actions/checkout@v4 - - - name: "Merge Changes Into ${{ inputs.target_branch }}" - uses: everlytic/branch-merge@1.1.5 - with: - source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} - target_branch: ${{ inputs.target_branch }} - github_token: ${{ secrets.FISHTOWN_BOT_PAT }} - commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" - - - name: "[Notification] Changes Merged into main" - run: | - title="Changelog and Version Bump Branch Merge" - message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into mains" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - determine-release-branch: - runs-on: ubuntu-latest - needs: - [ - create-temp-branch, - merge-changes-into-target-branch, - audit-changelog, - audit-version-in-code, - ] - # always run this job, regardless of if the dependant jobs were skipped - if: ${{ !failure() && !cancelled() }} - - # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, - # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a - # new sha to use from the merge we just did. Grab that here instead. - outputs: - final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} - - steps: - - name: "[Debug] Print Variables" - run: | - echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} - echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} - echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - - name: "Resolve Branch To Checkout" - id: resolve_branch - run: | - branch="" - if [ ${{ inputs.deploy_to == 'test' }}] || [ ${{ inputs.nightly_release == 'true' }} ] - then - branch=${{ needs.create-temp-branch.outputs.branch_name }} - else - branch="${{ inputs.target_branch }}" - fi - echo "target_branch=$branch" >> $GITHUB_OUTPUT - - name: "[Notification] Resolve Branch To Checkout" - run: | - title="Branch pick" - message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" - uses: actions/checkout@v4 - with: - ref: ${{ steps.resolve_branch.outputs.target_branch }} - - - name: "[Debug] Log Branch" - run: git status - - - name: "Resolve Commit SHA For Release" - id: resolve_commit_sha - run: | - echo "release_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" - if: ${{ inputs.deploy_to == 'prod' && inputs.nightly_release == 'false' && needs.create-temp-branch.outputs.branch_name != '' }} - run: | - git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} diff --git a/.github/workflows/resubmit-for-triage.yml b/.github/workflows/resubmit-for-triage.yml deleted file mode 100644 index 385ef820..00000000 --- a/.github/workflows/resubmit-for-triage.yml +++ /dev/null @@ -1,31 +0,0 @@ -# **what?** -# When triaging submissions, we sometimes need more information from the issue creator. -# In those cases we remove the `triage` label and add the `awaiting_response` label. -# Once we receive a response in the form of a comment, we want the `awaiting_response` label removed -# and the `triage` label added so that we are aware that the issue needs action. - -# **why?** -# This automates a part of issue triaging while also removing noise from triage lists. - -# **when?** -# This will run when a comment is added to an issue and that issue has an `awaiting_response` label. - -name: Resubmit for Triage - -on: issue_comment - -defaults: - run: - shell: bash - -permissions: - issues: write - -jobs: - triage_label: - if: contains(github.event.issue.labels.*.name, 'awaiting_response') - uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main - with: - add_label: "triage" - remove_label: "awaiting_response" - secrets: inherit # this is only acceptable because we own the action we're calling diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml deleted file mode 100644 index 75a14dd4..00000000 --- a/.github/workflows/stale.yml +++ /dev/null @@ -1,30 +0,0 @@ -# **what?** -# For issues that have been open for awhile without activity, label -# them as stale with a warning that they will be closed out. If -# anyone comments to keep the issue open, it will automatically -# remove the stale label and keep it open. - -# Stale label rules: -# awaiting_response, more_information_needed -> 90 days -# good_first_issue, help_wanted -> 360 days (a year) -# tech_debt -> 720 (2 years) -# all else defaults -> 180 days (6 months) - -# **why?** -# To keep the repo in a clean state from issues that aren't relevant anymore - -# **when?** -# Once a day - -name: "Close stale issues and PRs" -on: - schedule: - - cron: "30 1 * * *" - -permissions: - issues: write - pull-requests: write - -jobs: - stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/unit-tests.yml deleted file mode 100644 index b4ac615d..00000000 --- a/.github/workflows/unit-tests.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: Unit Tests - -on: - push: - branches: - - "main" - - "*.latest" - pull_request: - workflow_dispatch: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -jobs: - unit: - name: Unit Tests - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - with: - python-version: ${{ matrix.python-version }} - - - name: Run unit tests - run: hatch run unit-tests - shell: bash - - - name: Publish results - uses: ./.github/actions/publish-results - if: always() - with: - source-file: "results.csv" - file-name: "unit_results" - python-version: ${{ matrix.python-version }} From e71f0b3f1c2073330c77e4fec663c81284e798e7 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 17:45:58 -0500 Subject: [PATCH 03/63] update publish to work with dbt-adapters --- .github/workflows/_generate-changelog.yml | 229 ++++++++++++++++++++++ .github/workflows/_package-directory.yml | 26 +++ .github/workflows/_publish-internal.yml | 20 +- .github/workflows/_publish-pypi.yml | 12 +- .github/workflows/publish.yml | 14 +- 5 files changed, 287 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/_generate-changelog.yml create mode 100644 .github/workflows/_package-directory.yml diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml new file mode 100644 index 00000000..2b52864f --- /dev/null +++ b/.github/workflows/_generate-changelog.yml @@ -0,0 +1,229 @@ +name: "Changelog generation" + +on: + workflow_call: + inputs: + package: + description: "Choose the package getting published" + type: string + default: "dbt-adapters" + deploy-to: + description: "Choose whether we're publishing to test or prod" + type: string + default: "prod" + branch: + description: "Choose the branch to use" + type: string + default: "main" + outputs: + branch-name: + description: "The SHA to release" + value: ${{ jobs.branch.outputs.name }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + workflow_dispatch: + inputs: + package: + description: "Choose the package getting published" + type: string + default: "dbt-adapters" + deploy-to: + description: "Choose whether we're publishing to test or prod" + type: environment + default: "prod" + branch: + description: "Choose the branch to use" + type: string + default: "main" + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + + version: + needs: package + runs-on: ${{ vars.DEFAULT_RUNNER }} + outputs: + raw: ${{ steps.version.outputs.raw }} + base: ${{ steps.semver.outputs.base-version }} + prerelease: ${{ steps.semver.outputs.pre-release }} + is-prerelease: ${{ steps.semver.outputs.is-pre-release }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - uses: pypa/hatch@install + - id: version + run: echo "raw=$(hatch version)" >> $GITHUB_OUTPUT + working-directory: ./${{ needs.package.outputs.directory }} + - id: semver + uses: dbt-labs/actions/parse-semver@v1.1.1 + with: + version: ${{ steps.version.outputs.raw }} + + changelog: + runs-on: ${{ vars.DEFAULT_RUNNER }} + needs: [package, version] + outputs: + path: ${{ steps.changelog.outputs.path }} + exists: ${{ steps.changelog.outputs.exists }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - id: changelog + run: | + path=".changes/${{ needs.version.outputs.base }}" + if [[ ${{ needs.version.outputs.is-prerelease }} -eq 1 ]] + then + path+="-${{ needs.version.outputs.prerelease }}" + fi + path+=".md" + + echo "path=$path" >> $GITHUB_OUTPUT + + exists=false + if test -f $path + then + exists=true + fi + echo "exists=$exists">> $GITHUB_OUTPUT + working-directory: ./${{ needs.package.outputs.directory }} + + temp-branch: + needs: [version, changelog] + if: needs.changelog.outputs.exists == 'false' + runs-on: ${{ vars.DEFAULT_RUNNER }} + outputs: + name: ${{ steps.branch.outputs.name }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - id: branch + run: | + name="prep-release/${{ inputs.package }}/${{ inputs.deploy_to }}/${{ needs.version.outputs.raw }}_$GITHUB_RUN_ID" + echo "name=$name" >> $GITHUB_OUTPUT + - run: | + git checkout -b ${{ steps.branch.outputs.name }} + git push -u origin ${{ steps.branch.outputs.name }} + + dbt-membership: + needs: [version, changelog] + if: needs.changelog.outputs.exists == 'false' + runs-on: ${{ vars.DEFAULT_RUNNER }} + outputs: + team: ${{ steps.team.outputs.team }} + steps: + - id: temp-file + run: echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + - run: | + gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.temp-file.outputs.name }} + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + - id: team + run: | + team_list=$(jq -r '.[].login' ${{ steps.temp-file.outputs.name }}) + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "team=$team_list_single" >> $GITHUB_OUTPUT + - run: rm ${{ steps.temp-file.outputs.name }} + + generate-changelog: + needs: [package, version, changelog, temp-branch, dbt-membership] + if: needs.changelog.outputs.exists == 'false' + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.temp-branch.outputs.name }} + - run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + - run: | + brew install pre-commit + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + - run: | + if [[ ${{ needs.version.outputs.is-prerelease }} -eq 1 ]] + then + changie batch ${{ needs.version.outputs.base }} --move-dir '${{ needs.version.outputs.base }}' --prerelease ${{ needs.version.outputs.prerelease }} + elif [[ -d ".changes/${{ needs.version.outputs.base }}" ]] + then + changie batch ${{ needs.version.outputs.base }} --include '${{ needs.version.outputs.base }}' --remove-prereleases + else # releasing a final patch with no prereleases + changie batch ${{ needs.version.outputs.base }} + fi + changie merge + working-directory: ./${{ needs.package.outputs.directory }} + env: + CHANGIE_CORE_TEAM: ${{ needs.dbt-membership.outputs.team }} + - run: | + pre-commit run trailing-whitespace --files __version__.py CHANGELOG.md .changes/* + pre-commit run end-of-file-fixer --files __version__.py CHANGELOG.md .changes/* + working-directory: ./${{ needs.package.outputs.directory }} + continue-on-error: true + - run: | + git config user.name "Github Build Bot" + git config user.email "buildbot@fishtownanalytics.com" + git pull + git add . + git commit -m "generate changelog" + git push + working-directory: ./${{ needs.package.outputs.directory }} + + merge-changes: + runs-on: ${{ vars.DEFAULT_RUNNER }} + needs: [temp-branch, generate-changelog] + if: ${{ needs.temp-branch.outputs.name != '' && inputs.deploy-to == 'prod' }} + steps: + - uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.temp-branch.outputs.name }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + - run: git push origin -d ${{ needs.temp-branch.outputs.name }} + + branch: + needs: [temp-branch, merge-changes] + if: ${{ !failure() && !cancelled() }} + runs-on: ${{ vars.DEFAULT_RUNNER }} + # always run this job, regardless of whether changelog generation was skipped + # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, + # then it is what we will release. Otherwise, we generated a changelog and did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + name: ${{ steps.branch.outputs.name }} + steps: + - id: branch + run: | + branch="" + if [ ${{ needs.temp-branch.outputs.name == '' || inputs.deploy-to == 'prod' }}] + then + branch="${{ inputs.branch }}" + else + branch=${{ needs.temp-branch.outputs.name }} + fi + echo "name=$branch" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml new file mode 100644 index 00000000..11c43b3a --- /dev/null +++ b/.github/workflows/_package-directory.yml @@ -0,0 +1,26 @@ +name: "Package directory" + +on: + workflow_call: + inputs: + package: + description: "Choose the package whose directory you need" + type: string + default: "dbt-adapters" + +jobs: + package: + runs-on: ${{ vars.DEFAULT_RUNNER }} + outputs: + directory: ${{ steps.package.outputs.directory }} + steps: + - id: package + shell: bash + runs: | + if [[ package == 'dbt-adapters' ]] + then + directory="" + else + directory="${{ inputs.package }}" + fi + echo "directory=$directory" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml index 815c0f70..1c339701 100644 --- a/.github/workflows/_publish-internal.yml +++ b/.github/workflows/_publish-internal.yml @@ -35,7 +35,13 @@ defaults: shell: bash jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + publish: + needs: package runs-on: ${{ vars.DEFAULT_RUNNER }} environment: name: ${{ inputs.deploy-to }} @@ -57,7 +63,7 @@ jobs: # strip the pre-release off to find all iterations of this patch hatch version release echo "version=$(hatch version)" >> $GITHUB_OUTPUT - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} - id: published run: | versions_published="$(aws codeartifact list-package-versions \ @@ -73,14 +79,12 @@ jobs: with: version_number: ${{ steps.package.outputs.version }} versions_published: ${{ steps.published.outputs.versions }} - - name: "Update version to internal PyPI format" - run: | + - run: | VERSION=${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD) tee <<< "version = \"$VERSION\"" ./src/dbt/adapters/athena/__version__.py - working-directory: ./${{ inputs.package }} - - name: "Remove dbt-core from build requirements" - run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} + - run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml + working-directory: ./${{ needs.package.outputs.directory }} - run: | export HATCH_INDEX_USER=${{ secrets.AWS_USER }} @@ -99,4 +103,4 @@ jobs: hatch build --clean hatch run build:check-all hatch publish - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml index d8eed4c1..4150631d 100644 --- a/.github/workflows/_publish-pypi.yml +++ b/.github/workflows/_publish-pypi.yml @@ -38,7 +38,13 @@ defaults: shell: bash jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + publish: + needs: package runs-on: ${{ vars.DEFAULT_RUNNER }} environment: name: ${{ inputs.deploy-to }} @@ -58,7 +64,7 @@ jobs: # hatch will build using test PyPI first and fall back to prod PyPI when deploying to test # this is done via environment variables in the test environment in GitHub - run: hatch build && hatch run build:check-all - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} - uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: ${{ vars.PYPI_REPOSITORY_URL }} @@ -66,7 +72,7 @@ jobs: verify: runs-on: ${{ vars.DEFAULT_RUNNER }} - needs: publish + needs: [package, publish] # check the correct index environment: name: ${{ inputs.deploy-to }} @@ -76,7 +82,7 @@ jobs: ref: ${{ inputs.branch }} - id: version run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} - uses: nick-fields/retry@v3 with: timeout_seconds: 10 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 31558056..9a158097 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,14 +38,22 @@ jobs: package: ${{ inputs.package }} branch: ${{ inputs.branch }} + generate-changelog: + uses: ./.github/workflows/_generate-changelog.yml + with: + package: ${{ inputs.package }} + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} + secrets: inherit + publish-internal: if: ${{ inputs.pypi-internal == true }} - needs: unit-tests + needs: [unit-tests, generate-changelog] uses: ./.github/workflows/_publish-internal.yml with: package: ${{ inputs.package }} deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} + branch: ${{ needs.generate-changelog.outputs.branch-name }} secrets: inherit publish-pypi: @@ -55,4 +63,4 @@ jobs: with: package: ${{ inputs.package }} deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} + branch: ${{ needs.generate-changelog.outputs.branch-name }} From ff8fa55408e85f1e91d174032f2a65580018bd45 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 17:46:17 -0500 Subject: [PATCH 04/63] clean up existing workflows --- .github/workflows/changelog-exists.yml | 21 ++++++++++++++ .github/workflows/issue-triage.yml | 19 +++++++++++++ .github/workflows/scheduled-maintenance.yml | 24 ++++++++++++++++ .github/workflows/user-docs.yml | 31 +++++++++++++++++++++ 4 files changed, 95 insertions(+) create mode 100644 .github/workflows/changelog-exists.yml create mode 100644 .github/workflows/issue-triage.yml create mode 100644 .github/workflows/scheduled-maintenance.yml create mode 100644 .github/workflows/user-docs.yml diff --git a/.github/workflows/changelog-exists.yml b/.github/workflows/changelog-exists.yml new file mode 100644 index 00000000..1b5a2423 --- /dev/null +++ b/.github/workflows/changelog-exists.yml @@ -0,0 +1,21 @@ +name: "Check changelog entry exists" + +on: + pull_request_target: + types: [opened, reopened, labeled, unlabeled, synchronize] + +defaults: + run: + shell: bash + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' + skip_label: 'Skip Changelog' + secrets: inherit diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml new file mode 100644 index 00000000..5432121f --- /dev/null +++ b/.github/workflows/issue-triage.yml @@ -0,0 +1,19 @@ +name: "Issue triage" + +on: issue_comment + +defaults: + run: + shell: bash + +permissions: + issues: write + +jobs: + triage: + if: contains(github.event.issue.labels.*.name, 'triage:awaiting-response') + uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main + with: + add_label: "triage:dbt" + remove_label: "triage:awaiting-response" + secrets: inherit diff --git a/.github/workflows/scheduled-maintenance.yml b/.github/workflows/scheduled-maintenance.yml new file mode 100644 index 00000000..28309b95 --- /dev/null +++ b/.github/workflows/scheduled-maintenance.yml @@ -0,0 +1,24 @@ +name: "Scheduled maintenance" + +on: + schedule: + - cron: "30 1 * * SAT" + workflow_dispatch: + +permissions: + contents: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true + +jobs: + pre-commit-autoupdate: + uses: dbt-labs/actions/.github/workflows/pre-commit-autoupdate.yml + secrets: + TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} + SLACK_WEBHOOK_PR_URL: ${{ secrets.SLACK_DEV_ADAPTER_PULL_REQUESTS }} + SLACK_WEBHOOK_ALERTS_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} + + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/user-docs.yml b/.github/workflows/user-docs.yml new file mode 100644 index 00000000..66bd3da1 --- /dev/null +++ b/.github/workflows/user-docs.yml @@ -0,0 +1,31 @@ +name: "Open user docs issue" + +on: + issues: + types: [labeled, closed] + +defaults: + run: + shell: bash + +permissions: + issues: write # comments on issues + +jobs: + open_issues: + # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. + # If this logic does not exist in this workflow, it runs the + # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having + # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we + # decide if it should run or not. + if: | + (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( + (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || + (github.event.action == 'labeled' && github.event.label.name == 'user docs') + ) + uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main + with: + issue_repository: "dbt-labs/docs.getdbt.com" + issue_title: "Docs Changes Needed from dbt-adapters - Issue #${{ github.event.issue.number }}" + issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." + secrets: inherit From 607074e7f1f446824afb5de535c12e719e1528e6 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 17:46:40 -0500 Subject: [PATCH 05/63] update pull-request-checks to work with dbt-adapters --- .github/workflows/_unit-tests.yml | 1 - .github/workflows/_verify-build.yml | 73 +++++++++++++++++++++++ .github/workflows/pull-request-checks.yml | 20 +++++-- 3 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/_verify-build.yml diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index be2cac09..8e113b88 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -63,4 +63,3 @@ jobs: - uses: pypa/hatch@install - run: hatch run unit-tests shell: bash - working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml new file mode 100644 index 00000000..241b28c0 --- /dev/null +++ b/.github/workflows/_verify-build.yml @@ -0,0 +1,73 @@ +name: "Verify build" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to build" + type: string + default: "dbt-adapters" + branch: + description: "Choose the branch to build" + type: string + default: "main" + repository: + description: "Choose the repository to build, (used primarily when testing a fork)" + type: string + default: "dbt-labs/dbt-adapters" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: string + default: ${{ vars.DEFAULT_PYTHON_VERSION }} + workflow_dispatch: + inputs: + package: + description: "Choose the package to build" + type: choice + options: ["dbt-adapters", "dbt-tests-adapter"] + branch: + description: "Choose the branch to build" + type: string + default: "main" + repository: + description: "Choose the repository to build, (used primarily when testing a fork)" + type: string + default: "dbt-labs/dbt-adapters" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: choice + options: ["3.9", "3.10", "3.11", "3.12"] + +permissions: read-all + +defaults: + run: + shell: bash + +jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + + build: + needs: package + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - uses: pypa/hatch@install + - run: hatch build && hatch run build:check-all + working-directory: ./${{ needs.package.outputs.directory }} diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 2322ff91..770deef3 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -16,21 +16,33 @@ jobs: branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - unit-tests: - uses: ./.github/workflows/_unit-tests.yml + verify-builds: + uses: ./.github/workflows/_verify-build.yml + strategy: + matrix: + package: ["dbt-adapters", "dbt-tests-adapter"] + python-version: ["3.9", "3.10", "3.11", "3.12"] with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} + python-version: ${{ matrix.python-version }} + + unit-tests: + uses: ./.github/workflows/_unit-tests.yml strategy: matrix: package: ["dbt-adapters"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} # This job does nothing and is only used for branch protection results: - name: "Pull request checks" + name: "Pull request checks" # keep this name, branch protection references it if: always() - needs: [code-quality, unit-tests] + needs: [code-quality, verify-builds, unit-tests] runs-on: ${{ vars.DEFAULT_RUNNER }} steps: - uses: re-actors/alls-green@release/v1 From f5dfafc62b20cc89818bad654811c66e90398927 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:12:28 -0500 Subject: [PATCH 06/63] add pull_request target for checking workflows --- .github/workflows/pull-request-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 770deef3..ded0f443 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,6 +1,8 @@ name: "Pull request checks" on: + pull_request: + types: [opened, reopened, synchronize] pull_request_target: types: [opened, reopened, synchronize] From 07dd825f758d829180fbf37249a717c2375b95f0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:16:49 -0500 Subject: [PATCH 07/63] fix typo in _package-directory.yml --- .github/workflows/_package-directory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 11c43b3a..115abdfe 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -16,7 +16,7 @@ jobs: steps: - id: package shell: bash - runs: | + run: | if [[ package == 'dbt-adapters' ]] then directory="" From 997404fdee31e486f9111aab3964081ea67b195b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:23:22 -0500 Subject: [PATCH 08/63] remove env vars from inputs --- .github/workflows/_unit-tests.yml | 6 +++--- .github/workflows/_verify-build.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index 8e113b88..7e6b40f8 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -18,11 +18,11 @@ on: os: description: "Choose the OS to test against" type: string - default: ${{ vars.DEFAULT_RUNNER }} + default: "ubuntu-22.04" python-version: description: "Choose the Python version to test against" type: string - default: ${{ vars.DEFAULT_PYTHON_VERSION }} + default: 3.9 workflow_dispatch: inputs: package: @@ -40,7 +40,7 @@ on: os: description: "Choose the OS to test against" type: string - default: ${{ vars.DEFAULT_RUNNER }} + default: "ubuntu-22.04" python-version: description: "Choose the Python version to test against" type: choice diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml index 241b28c0..a5c33030 100644 --- a/.github/workflows/_verify-build.yml +++ b/.github/workflows/_verify-build.yml @@ -18,11 +18,11 @@ on: os: description: "Choose the OS to test against" type: string - default: ${{ vars.DEFAULT_RUNNER }} + default: "ubuntu-22.04" python-version: description: "Choose the Python version to test against" type: string - default: ${{ vars.DEFAULT_PYTHON_VERSION }} + default: "3.9" workflow_dispatch: inputs: package: @@ -40,7 +40,7 @@ on: os: description: "Choose the OS to test against" type: string - default: ${{ vars.DEFAULT_RUNNER }} + default: "ubuntu-22.04" python-version: description: "Choose the Python version to test against" type: choice From ea681912d11c467fb9bbbb2c31c592368230ed3d Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:28:27 -0500 Subject: [PATCH 09/63] add python version to unit test matrix --- .github/workflows/pull-request-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index ded0f443..e6c7de36 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -35,6 +35,7 @@ jobs: strategy: matrix: package: ["dbt-adapters"] + python-version: ["3.9", "3.10", "3.11", "3.12"] with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} From e424dfc5fab46551096cfbba69e26bcab02e1cd9 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:57:05 -0500 Subject: [PATCH 10/63] register workflow with gha --- .github/workflows/publish.yml | 9 +++++---- pyproject.toml | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9a158097..e7d19014 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,7 @@ name: "Publish" on: + pull_request: workflow_dispatch: inputs: package: @@ -12,19 +13,19 @@ on: deploy-to: description: "Choose whether to publish to test or prod" type: environment - default: "prod" + default: "test" branch: description: "Choose the branch to publish from" type: string - default: "main" + default: "monorepo/shared-workflows" pypi-internal: description: "Publish Internally" type: boolean - default: true + default: false pypi-public: description: "Publish to PyPI" type: boolean - default: false + default: true # don't attempt to release the same target in parallel concurrency: diff --git a/pyproject.toml b/pyproject.toml index 3c62898d..94d0060b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" +test-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" [tool.hatch.envs.build] detached = true From 8ba75f7645d5550afa2909859e4cfcafb56ee53e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:58:05 -0500 Subject: [PATCH 11/63] revert defaults --- .github/workflows/publish.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e7d19014..9a158097 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,7 +1,6 @@ name: "Publish" on: - pull_request: workflow_dispatch: inputs: package: @@ -13,19 +12,19 @@ on: deploy-to: description: "Choose whether to publish to test or prod" type: environment - default: "test" + default: "prod" branch: description: "Choose the branch to publish from" type: string - default: "monorepo/shared-workflows" + default: "main" pypi-internal: description: "Publish Internally" type: boolean - default: false + default: true pypi-public: description: "Publish to PyPI" type: boolean - default: true + default: false # don't attempt to release the same target in parallel concurrency: From 76a18a6114f5b9d75ac9db49b52820b691240332 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:12:14 -0500 Subject: [PATCH 12/63] update workflow deps for publish.yml --- .github/workflows/publish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9a158097..5fe98574 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,6 +39,7 @@ jobs: branch: ${{ inputs.branch }} generate-changelog: + needs: unit-tests uses: ./.github/workflows/_generate-changelog.yml with: package: ${{ inputs.package }} @@ -48,7 +49,7 @@ jobs: publish-internal: if: ${{ inputs.pypi-internal == true }} - needs: [unit-tests, generate-changelog] + needs: generate-changelog uses: ./.github/workflows/_publish-internal.yml with: package: ${{ inputs.package }} @@ -58,7 +59,7 @@ jobs: publish-pypi: if: ${{ inputs.pypi-public == true }} - needs: unit-tests + needs: generate-changelog uses: ./.github/workflows/_publish-pypi.yml with: package: ${{ inputs.package }} From 139bcf819cf3f9998a5794ce24179fba3fdff7d8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:15:52 -0500 Subject: [PATCH 13/63] bump version just to test changelog generation --- dbt/adapters/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 8c657eec..57ac4b25 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.10.2" +version = "1.10.3a1" From d16ec9738c444658681d15322aba3eae644205c8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:22:33 -0500 Subject: [PATCH 14/63] fix temp branch name for changelog generation --- .github/workflows/_generate-changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml index 2b52864f..aca712b2 100644 --- a/.github/workflows/_generate-changelog.yml +++ b/.github/workflows/_generate-changelog.yml @@ -126,7 +126,7 @@ jobs: ref: ${{ inputs.branch }} - id: branch run: | - name="prep-release/${{ inputs.package }}/${{ inputs.deploy_to }}/${{ needs.version.outputs.raw }}_$GITHUB_RUN_ID" + name="prep-release/${{ inputs.package }}/$GITHUB_RUN_ID" echo "name=$name" >> $GITHUB_OUTPUT - run: | git checkout -b ${{ steps.branch.outputs.name }} From cdecc9dd5638fafca6b62c4ac49dee886c2e565b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:35:49 -0500 Subject: [PATCH 15/63] revert version bump used for testing publish action --- dbt/adapters/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 57ac4b25..8c657eec 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.10.3a1" +version = "1.10.2" From 7080796ed098615e08fc3c4502c0fd2db08e22bb Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:37:02 -0500 Subject: [PATCH 16/63] revert pull_request trigger that was used for testing --- .github/workflows/pull-request-checks.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index e6c7de36..354995d5 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,8 +1,6 @@ name: "Pull request checks" on: - pull_request: - types: [opened, reopened, synchronize] pull_request_target: types: [opened, reopened, synchronize] From 87f439a2f1158fe5f60fdd588077d0b9d6b98997 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 4 Dec 2024 20:31:39 -0500 Subject: [PATCH 17/63] add in changelog entry check --- .github/workflows/_changelog-entry-check.yml | 57 ++++++++++++++++++++ .github/workflows/pull-request-checks.yml | 7 ++- 2 files changed, 63 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/_changelog-entry-check.yml diff --git a/.github/workflows/_changelog-entry-check.yml b/.github/workflows/_changelog-entry-check.yml new file mode 100644 index 00000000..93f453c0 --- /dev/null +++ b/.github/workflows/_changelog-entry-check.yml @@ -0,0 +1,57 @@ +name: "Changelog entry check" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to test" + type: string + default: "dbt-adapters" + issue: + description: "The issue/PR number" + type: string + required: true + +permissions: + contents: read + pull-requests: write + +jobs: + changelog-check: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'Skip Changelog') }} + outputs: + exists: ${{ steps.changelog.outputs.exists }} + runs-on: ubuntu-latest + steps: + - id: changelog + uses: dorny/paths-filter@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + filters: | + exists: + - added|modified: '${{ inputs.package }}/.changes/unreleased/**.yaml' + + comment: + needs: changelog-check + if: needs.changelog-check.outputs.exists == false + runs-on: ubuntu-latest + env: + COMMENT: >- + Thank you for your pull request! We could not find a changelog entry for this change. + For details on how to document a change, see the + [Contributing Guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md). + steps: + - id: comment + uses: peter-evans/find-comment@v3 + with: + issue-number: ${{ inputs.issue }} + comment-author: "github-actions[bot]" + body-includes: ${{ env.COMMENT }} + - if: steps.comment.outputs.comment-body == '' + run: gh issue comment ${{ inputs.issue }} --repo ${{ github.repository }} --body "${{ env.COMMENT }}" + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/github-script@v7 + with: + script: core.setFailed('Changelog entry required to merge.') diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 354995d5..7f345a04 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -2,7 +2,7 @@ name: "Pull request checks" on: pull_request_target: - types: [opened, reopened, synchronize] + types: [opened, reopened, synchronize, labeled, unlabeled] # only run this once per PR at a time concurrency: @@ -10,6 +10,11 @@ concurrency: cancel-in-progress: true jobs: + changelog-entry: + uses: ./.github/workflows/_changelog-entry-check.yml + with: + issue: ${{ github.event.pull_request.number }} + code-quality: uses: ./.github/workflows/_code-quality.yml with: From bcaaa09a449c366ba0ea04f9805696ac518c4e54 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 6 Dec 2024 17:37:45 -0500 Subject: [PATCH 18/63] rename files for git rebase --- .../{changelog-existence.yml => _changelog-entry-check.yml} | 0 .github/workflows/{code-quality.yml => _code-quality.yml} | 0 .github/workflows/{unit-tests.yml => _unit-tests.yml} | 0 .github/workflows/{resubmit-for-triage.yml => issue-triage.yml} | 0 .github/workflows/{release.yml => publish.yml} | 0 .github/workflows/{stale.yml => scheduled-maintenance.yml} | 0 .github/workflows/{docs-issue.yml => user-docs.yml} | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{changelog-existence.yml => _changelog-entry-check.yml} (100%) rename .github/workflows/{code-quality.yml => _code-quality.yml} (100%) rename .github/workflows/{unit-tests.yml => _unit-tests.yml} (100%) rename .github/workflows/{resubmit-for-triage.yml => issue-triage.yml} (100%) rename .github/workflows/{release.yml => publish.yml} (100%) rename .github/workflows/{stale.yml => scheduled-maintenance.yml} (100%) rename .github/workflows/{docs-issue.yml => user-docs.yml} (100%) diff --git a/.github/workflows/changelog-existence.yml b/.github/workflows/_changelog-entry-check.yml similarity index 100% rename from .github/workflows/changelog-existence.yml rename to .github/workflows/_changelog-entry-check.yml diff --git a/.github/workflows/code-quality.yml b/.github/workflows/_code-quality.yml similarity index 100% rename from .github/workflows/code-quality.yml rename to .github/workflows/_code-quality.yml diff --git a/.github/workflows/unit-tests.yml b/.github/workflows/_unit-tests.yml similarity index 100% rename from .github/workflows/unit-tests.yml rename to .github/workflows/_unit-tests.yml diff --git a/.github/workflows/resubmit-for-triage.yml b/.github/workflows/issue-triage.yml similarity index 100% rename from .github/workflows/resubmit-for-triage.yml rename to .github/workflows/issue-triage.yml diff --git a/.github/workflows/release.yml b/.github/workflows/publish.yml similarity index 100% rename from .github/workflows/release.yml rename to .github/workflows/publish.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/scheduled-maintenance.yml similarity index 100% rename from .github/workflows/stale.yml rename to .github/workflows/scheduled-maintenance.yml diff --git a/.github/workflows/docs-issue.yml b/.github/workflows/user-docs.yml similarity index 100% rename from .github/workflows/docs-issue.yml rename to .github/workflows/user-docs.yml From c527fe93d089a9288e6a33415340088ee7f6e2e7 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 15:42:48 -0500 Subject: [PATCH 19/63] create new batch of workflows --- .github/workflows/_code-quality.yml | 62 +++--- .github/workflows/_publish-internal.yml | 102 ++++++++++ .github/workflows/_publish-pypi.yml | 85 +++++++++ .github/workflows/_unit-tests.yml | 105 ++++++----- .github/workflows/publish.yml | 220 +++++----------------- .github/workflows/pull-request-checks.yml | 38 ++++ 6 files changed, 369 insertions(+), 243 deletions(-) create mode 100644 .github/workflows/_publish-internal.yml create mode 100644 .github/workflows/_publish-pypi.yml create mode 100644 .github/workflows/pull-request-checks.yml diff --git a/.github/workflows/_code-quality.yml b/.github/workflows/_code-quality.yml index 9c203847..d3ddfc05 100644 --- a/.github/workflows/_code-quality.yml +++ b/.github/workflows/_code-quality.yml @@ -1,34 +1,40 @@ -name: Code Quality +name: "Code quality" on: - push: - branches: - - "main" - - "*.latest" - pull_request: workflow_dispatch: + workflow_call: + inputs: + branch: + description: "Choose the branch to check" + type: string + default: "main" + repository: + description: "Choose the repository to check, when using a fork" + type: string + default: "dbt-labs/dbt-adapters" + workflow_dispatch: + inputs: + branch: + description: "Choose the branch to check" + type: string + default: "main" + repository: + description: "Choose the repository to check, when using a fork" + type: string + default: "dbt-labs/dbt-adapters" -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true +permissions: + contents: read jobs: - code-quality: - name: Code Quality - runs-on: ubuntu-latest - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Run code quality - shell: bash - run: hatch run code-quality + code-quality: + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + repository: ${{ inputs.repository }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - uses: pre-commit/action@v3.0.1 diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml new file mode 100644 index 00000000..815c0f70 --- /dev/null +++ b/.github/workflows/_publish-internal.yml @@ -0,0 +1,102 @@ +name: "Publish internally" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to publish" + type: string + default: "dbt-adapters" + deploy-to: + description: "Choose whether to publish to test or prod" + type: string + default: "prod" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + workflow_dispatch: + inputs: + package: + description: "Choose the package to publish" + type: choice + options: ["dbt-adapters"] + deploy-to: + description: "Choose whether to publish to test or prod" + type: environment + default: "test" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + +defaults: + run: + shell: bash + +jobs: + publish: + runs-on: ${{ vars.DEFAULT_RUNNER }} + environment: + name: ${{ inputs.deploy-to }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - uses: pypa/hatch@install + - uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: ${{ vars.AWS_REGION }} + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + - id: package + run: | + # strip the pre-release off to find all iterations of this patch + hatch version release + echo "version=$(hatch version)" >> $GITHUB_OUTPUT + working-directory: ./${{ inputs.package }} + - id: published + run: | + versions_published="$(aws codeartifact list-package-versions \ + --domain ${{ vars.AWS_DOMAIN }} \ + --repository ${{ vars.AWS_REPOSITORY }} \ + --format pypi \ + --package ${{ inputs.package }} \ + --output json \ + --query 'versions[*].version' | jq -r '.[]' | grep "^${{ steps.package.outputs.version }}" || true )" # suppress pipefail only here + echo "versions=$(echo "${versions_published[*]}"| tr '\n' ',')" >> $GITHUB_OUTPUT + - id: next + uses: dbt-labs/dbt-release/.github/actions/next-cloud-release-version@main + with: + version_number: ${{ steps.package.outputs.version }} + versions_published: ${{ steps.published.outputs.versions }} + - name: "Update version to internal PyPI format" + run: | + VERSION=${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD) + tee <<< "version = \"$VERSION\"" ./src/dbt/adapters/athena/__version__.py + working-directory: ./${{ inputs.package }} + - name: "Remove dbt-core from build requirements" + run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml + working-directory: ./${{ inputs.package }} + - run: | + export HATCH_INDEX_USER=${{ secrets.AWS_USER }} + + export HATCH_INDEX_AUTH=$(aws codeartifact get-authorization-token \ + --domain ${{ vars.AWS_DOMAIN }} \ + --output text \ + --query authorizationToken) + + export HATCH_INDEX_REPO=$(aws codeartifact get-repository-endpoint \ + --domain ${{ vars.AWS_DOMAIN }} \ + --repository ${{ vars.AWS_REPOSITORY }} \ + --format pypi \ + --output text \ + --query repositoryEndpoint) + + hatch build --clean + hatch run build:check-all + hatch publish + working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml new file mode 100644 index 00000000..d8eed4c1 --- /dev/null +++ b/.github/workflows/_publish-pypi.yml @@ -0,0 +1,85 @@ +name: "Publish to PyPI" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to publish" + type: string + default: "dbt-adapters" + deploy-to: + description: "Choose whether to publish to test or prod" + type: string + default: "prod" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + workflow_dispatch: + inputs: + package: + description: "Choose the package to publish" + type: choice + options: ["dbt-adapters"] + deploy-to: + description: "Choose whether to publish to test or prod" + type: environment + default: "test" + branch: + description: "Choose the branch to publish" + type: string + default: "main" + +permissions: + contents: read + +defaults: + run: + shell: bash + +jobs: + publish: + runs-on: ${{ vars.DEFAULT_RUNNER }} + environment: + name: ${{ inputs.deploy-to }} + url: ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }} + permissions: + # this permission is required for trusted publishing + # see https://github.com/marketplace/actions/pypi-publish + id-token: write + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - uses: pypa/hatch@install + # hatch will build using test PyPI first and fall back to prod PyPI when deploying to test + # this is done via environment variables in the test environment in GitHub + - run: hatch build && hatch run build:check-all + working-directory: ./${{ inputs.package }} + - uses: pypa/gh-action-pypi-publish@release/v1 + with: + repository-url: ${{ vars.PYPI_REPOSITORY_URL }} + packages-dir: ${{ inputs.package }}/dist/ + + verify: + runs-on: ${{ vars.DEFAULT_RUNNER }} + needs: publish + # check the correct index + environment: + name: ${{ inputs.deploy-to }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - id: version + run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT + working-directory: ./${{ inputs.package }} + - uses: nick-fields/retry@v3 + with: + timeout_seconds: 10 + retry_wait_seconds: 10 + max_attempts: 15 # 5 minutes: (10s timeout + 10s delay) * 15 attempts + command: wget ${{ vars.PYPI_PROJECT_URL }}/${{ inputs.package }}/${{ steps.version.outputs.version }} diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index b4ac615d..be2cac09 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -1,49 +1,66 @@ -name: Unit Tests +name: "Unit tests" on: - push: - branches: - - "main" - - "*.latest" - pull_request: - workflow_dispatch: + workflow_call: + inputs: + package: + description: "Choose the package to test" + type: string + default: "dbt-adapters" + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-adapters" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: string + default: ${{ vars.DEFAULT_PYTHON_VERSION }} + workflow_dispatch: + inputs: + package: + description: "Choose the package to test" + type: choice + options: ["dbt-adapters"] + branch: + description: "Choose the branch to test" + type: string + default: "main" + repository: + description: "Choose the repository to test, when using a fork" + type: string + default: "dbt-labs/dbt-adapters" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: choice + options: ["3.9", "3.10", "3.11", "3.12"] -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true +permissions: + contents: read jobs: - unit: - name: Unit Tests - runs-on: ubuntu-latest - - strategy: - fail-fast: false - matrix: - python-version: ["3.9", "3.10", "3.11", "3.12"] - - steps: - - name: Check out repository - uses: actions/checkout@v4 - with: - persist-credentials: false - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - with: - python-version: ${{ matrix.python-version }} - - - name: Run unit tests - run: hatch run unit-tests - shell: bash - - - name: Publish results - uses: ./.github/actions/publish-results - if: always() - with: - source-file: "results.csv" - file-name: "unit_results" - python-version: ${{ matrix.python-version }} + unit-tests: + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + repository: ${{ inputs.repository }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - uses: pypa/hatch@install + - run: hatch run unit-tests + shell: bash + working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 828350dd..31558056 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,180 +1,58 @@ -name: Release -run-name: Release ${{ inputs.package }}==${{ inputs.version_number }} to ${{ inputs.deploy-to }} +name: "Publish" on: - workflow_dispatch: - inputs: - package: - type: choice - description: Choose what to publish - options: - - dbt-adapters - - dbt-tests-adapter - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - deploy-to: - type: choice - description: Choose where to publish - options: - - prod - - test - default: prod - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - - workflow_call: - inputs: - package: - type: string - description: Choose what to publish - required: true - version_number: - description: "The release version number (i.e. 1.0.0b1)" - type: string - required: true - deploy-to: - type: string - default: prod - required: false - nightly_release: - description: "Nightly release to dev environment" - type: boolean - default: false - required: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - -# this is the permission that allows creating a new release -permissions: - contents: write - id-token: write - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise + workflow_dispatch: + inputs: + package: + description: "Choose the package to publish" + type: choice + options: + - "dbt-adapters" + - "dbt-tests-adapter" + deploy-to: + description: "Choose whether to publish to test or prod" + type: environment + default: "prod" + branch: + description: "Choose the branch to publish from" + type: string + default: "main" + pypi-internal: + description: "Publish Internally" + type: boolean + default: true + pypi-public: + description: "Publish to PyPI" + type: boolean + default: false + +# don't attempt to release the same target in parallel concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }}-${{ inputs.package }}-${{ inputs.deploy-to }} - cancel-in-progress: true - -defaults: - run: - shell: bash + group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.deploy-to }} + cancel-in-progress: true jobs: - release-inputs: - name: "Release inputs" - runs-on: ubuntu-latest - outputs: - working-dir: ${{ steps.release-inputs.outputs.working-dir }} - run-unit-tests: ${{ steps.release-inputs.outputs.run-unit-tests }} - archive-name: ${{ steps.release-inputs.outputs.archive-name }} - steps: - - name: "Inputs" - id: release-inputs - run: | - working_dir="./" - run_unit_tests=true - archive_name=${{ inputs.package }}-${{ inputs.version_number }}-${{ inputs.deploy-to }} - - if test "${{ inputs.package }}" = "dbt-tests-adapter" - then - working_dir="./dbt-tests-adapter/" - run_unit_tests=false - fi - - echo "working-dir=$working_dir" >> $GITHUB_OUTPUT - echo "run-unit-tests=$run_unit_tests" >> $GITHUB_OUTPUT - echo "archive-name=$archive_name" >> $GITHUB_OUTPUT - - - name: "[DEBUG]" - run: | - echo package : ${{ inputs.package }} - echo working-dir : ${{ steps.release-inputs.outputs.working-dir }} - echo run-unit-tests : ${{ steps.release-inputs.outputs.run-unit-tests }} - echo archive-name : ${{ steps.release-inputs.outputs.archive-name }} - - bump-version-generate-changelog: - name: "Bump package version, Generate changelog" - uses: dbt-labs/dbt-adapters/.github/workflows/release_prep_hatch.yml@main - needs: [release-inputs] - with: - version_number: ${{ inputs.version_number }} - deploy_to: ${{ inputs.deploy-to }} - nightly_release: ${{ inputs.nightly_release }} - target_branch: ${{ inputs.target_branch }} - working-dir: ${{ needs.release-inputs.outputs.working-dir }} - run-unit-tests: ${{ fromJSON(needs.release-inputs.outputs.run-unit-tests) }} - secrets: inherit - - log-outputs-bump-version-generate-changelog: - name: "[Log output] Bump package version, Generate changelog" - if: ${{ !failure() && !cancelled() }} - needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - steps: - - name: Print variables - run: | - echo Final SHA : ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - echo Changelog path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - - build-and-test: - name: "Build and Test" - needs: [release-inputs, bump-version-generate-changelog] - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out repository" - uses: actions/checkout@v4 + unit-tests: + uses: ./.github/workflows/_unit-tests.yml with: - ref: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - - - name: "Setup `hatch`" - uses: dbt-labs/dbt-adapters/.github/actions/setup-hatch@main + package: ${{ inputs.package }} + branch: ${{ inputs.branch }} - - name: "Build ${{ inputs.package }}" - uses: dbt-labs/dbt-adapters/.github/actions/build-hatch@main + publish-internal: + if: ${{ inputs.pypi-internal == true }} + needs: unit-tests + uses: ./.github/workflows/_publish-internal.yml with: - working-dir: ${{ needs.release-inputs.outputs.working-dir }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} - - github-release: - name: "GitHub Release" - # ToDo: update GH release to handle adding dbt-tests-adapter and dbt-adapters assets to the same release - if: ${{ !failure() && !cancelled() && inputs.package == 'dbt-adapters' }} - needs: [release-inputs, build-and-test, bump-version-generate-changelog] - uses: dbt-labs/dbt-adapters/.github/workflows/github-release.yml@main - with: - sha: ${{ needs.bump-version-generate-changelog.outputs.final_sha }} - version_number: ${{ inputs.version_number }} - changelog_path: ${{ needs.bump-version-generate-changelog.outputs.changelog_path }} - test_run: ${{ inputs.deploy-to == 'test' && true || false }} - archive_name: ${{ needs.release-inputs.outputs.archive-name }} - - pypi-release: - name: "Publish to PyPI" - runs-on: ubuntu-latest - needs: [release-inputs, build-and-test] - environment: - name: ${{ inputs.deploy-to }} - url: ${{ vars.PYPI_PROJECT_URL }} - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - - - name: "Publish to PyPI" - uses: dbt-labs/dbt-adapters/.github/actions/publish-pypi@main + package: ${{ inputs.package }} + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} + secrets: inherit + + publish-pypi: + if: ${{ inputs.pypi-public == true }} + needs: unit-tests + uses: ./.github/workflows/_publish-pypi.yml with: - repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - archive-name: ${{ needs.release-inputs.outputs.archive-name }} + package: ${{ inputs.package }} + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml new file mode 100644 index 00000000..2322ff91 --- /dev/null +++ b/.github/workflows/pull-request-checks.yml @@ -0,0 +1,38 @@ +name: "Pull request checks" + +on: + pull_request_target: + types: [opened, reopened, synchronize] + +# only run this once per PR at a time +concurrency: + group: ${{ github.workflow }}-${{ github.event.number }} + cancel-in-progress: true + +jobs: + code-quality: + uses: ./.github/workflows/_code-quality.yml + with: + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + + unit-tests: + uses: ./.github/workflows/_unit-tests.yml + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + strategy: + matrix: + package: ["dbt-adapters"] + + # This job does nothing and is only used for branch protection + results: + name: "Pull request checks" + if: always() + needs: [code-quality, unit-tests] + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + - uses: re-actors/alls-green@release/v1 + with: + jobs: ${{ toJSON(needs) }} From f7586aa9d91387062ba59b1f8e864d3b5323db4e Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 17:45:30 -0500 Subject: [PATCH 20/63] remove old workflows --- .github/workflows/build.yml | 54 -- .github/workflows/github-release.yml | 259 ---------- .github/workflows/precommit-autoupdate.yml | 22 - .github/workflows/release_prep_hatch.yml | 542 --------------------- 4 files changed, 877 deletions(-) delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/github-release.yml delete mode 100644 .github/workflows/precommit-autoupdate.yml delete mode 100644 .github/workflows/release_prep_hatch.yml diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 00afd704..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,54 +0,0 @@ -# **what?** -# Verifies python build on all code commited to the repository. This workflow -# should not require any secrets since it runs for PRs from forked repos. By -# default, secrets are not passed to workflows running from a forked repos. - -# **why?** -# Ensure code for dbt meets a certain quality standard. - -# **when?** -# This will run for all PRs, when code is pushed to main, and when manually triggered. - -name: "Build" - -on: - push: - branches: - - "main" - pull_request: - merge_group: - types: [checks_requested] - workflow_dispatch: - workflow_call: - -permissions: read-all - -# will cancel previous workflows triggered by the same event and for the same ref for PRs or same SHA otherwise -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ contains(github.event_name, 'pull_request') && github.event.pull_request.head.ref || github.sha }} - cancel-in-progress: true - -defaults: - run: - shell: bash - -jobs: - build: - name: Build, Test and publish to PyPi - runs-on: ubuntu-latest - permissions: - id-token: write # IMPORTANT: this permission is mandatory for trusted publishing - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: Build `dbt-adapters` - uses: ./.github/actions/build-hatch - - - name: Build `dbt-tests-adapter` - uses: ./.github/actions/build-hatch - with: - working-dir: "./dbt-tests-adapter/" diff --git a/.github/workflows/github-release.yml b/.github/workflows/github-release.yml deleted file mode 100644 index ad0cc2d8..00000000 --- a/.github/workflows/github-release.yml +++ /dev/null @@ -1,259 +0,0 @@ -# **what?** -# Create a new release on GitHub and include any artifacts in the `/dist` directory of the GitHub artifacts store. -# -# Inputs: -# sha: The commit to attach to this release -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# changelog_path: Path to the changelog file for release notes -# test_run: Test run (Publish release as draft) -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call after a successful build. Build artifacts should be ready to release and live in a dist/ directory. -# -# This workflow expects the artifacts to already be built and living in the artifact store of the workflow. -# -# Validation Checks -# -# 1. If no release already exists for this commit and version, create the tag and release it to GitHub. -# 2. If a release already exists for this commit, skip creating the release but finish with a success. -# 3. If a release exists for this commit under a different tag, fail. -# 4. If the commit is already associated with a different release, fail. - -name: GitHub Release - -on: - workflow_call: - inputs: - sha: - description: The commit to attach to this release - required: true - type: string - version_number: - description: The release version number (i.e. 1.0.0b1) - required: true - type: string - changelog_path: - description: Path to the changelog file for release notes - required: true - type: string - test_run: - description: Test run (Publish release as draft) - required: true - type: boolean - archive_name: - description: artifact name to download - required: true - type: string - outputs: - tag: - description: The path to the changelog for this version - value: ${{ jobs.check-release-exists.outputs.tag }} - -permissions: - contents: write - -env: - REPO_LINK: ${{ github.server_url }}/${{ github.repository }} - NOTIFICATION_PREFIX: "[GitHub Release]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - steps: - - name: "[DEBUG] Print Variables" - run: | - echo The last commit sha in the release: ${{ inputs.sha }} - echo The release version number: ${{ inputs.version_number }} - echo Expected Changelog path: ${{ inputs.changelog_path }} - echo Test run: ${{ inputs.test_run }} - echo Repo link: ${{ env.REPO_LINK }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - - check-release-exists: - runs-on: ubuntu-latest - outputs: - exists: ${{ steps.release_check.outputs.exists }} - draft_exists: ${{ steps.release_check.outputs.draft_exists }} - tag: ${{ steps.set_tag.outputs.tag }} - - steps: - - name: "Generate Release Tag" - id: set_tag - run: echo "tag=v${{ inputs.version_number }}" >> $GITHUB_OUTPUT - - # When the GitHub CLI doesn't find a release for the given tag, it will exit 1 with a - # message of "release not found". In our case, it's not an actual error, just a - # confirmation that the release does not already exists so we can go ahead and create it. - # The `|| true` makes it so the step does not exit with a non-zero exit code - # Also check if the release already exists is draft state. If it does, and we are not - # testing then we can publish that draft as is. If it's in draft and we are testing, skip the - # release. - - name: "Check If Release Exists For Tag ${{ steps.set_tag.outputs.tag }}" - id: release_check - run: | - output=$((gh release view ${{ steps.set_tag.outputs.tag }} --json isDraft,targetCommitish --repo ${{ env.REPO_LINK }}) 2>&1) || true - if [[ "$output" == "release not found" ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} does not exist." - message="Check passed." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - commit=$(jq -r '.targetCommitish' <<< "$output") - if [[ $commit != ${{ inputs.sha }} ]] - then - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists for commit $commit!" - message="Cannot create a new release for commit ${{ inputs.sha }}. Exiting." - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - isDraft=$(jq -r '.isDraft' <<< "$output") - if [[ $isDraft == true ]] && [[ ${{ inputs.test_run }} == false ]] - then - title="Release tag ${{ steps.set_tag.outputs.tag }} already associated with the draft release." - message="Release workflow will publish the associated release." - echo "exists=false" >> $GITHUB_OUTPUT - echo "draft_exists=true" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 0 - fi - title="Release for tag ${{ steps.set_tag.outputs.tag }} already exists." - message="Skip GitHub Release Publishing." - echo "exists=true" >> $GITHUB_OUTPUT - echo "draft_exists=false" >> $GITHUB_OUTPUT - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO: ${{ env.REPO_LINK }} - - - name: "[DEBUG] Log Job Outputs" - run: | - echo exists: ${{ steps.release_check.outputs.exists }} - echo draft_exists: ${{ steps.release_check.outputs.draft_exists }} - echo tag: ${{ steps.set_tag.outputs.tag }} - - skip-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'true' - - steps: - - name: "Tag Exists, Skip GitHub Release Job" - run: | - echo title="A tag already exists for ${{ needs.check-release-exists.outputs.tag }} and commit." - echo message="Skipping GitHub release." - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - audit-release-different-commit: - runs-on: ubuntu-latest - needs: [check-release-exists] - if: needs.check-release-exists.outputs.exists == 'false' - - steps: - - name: "Check If Release Already Exists For Commit" - uses: cardinalby/git-get-release-action@1.2.4 - id: check_release_commit - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - with: - commitSha: ${{ inputs.sha }} - doNotFailIfNotFound: true # returns blank outputs when not found instead of error - searchLimit: 15 # Since we only care about recent releases, speed up the process - - - name: "[DEBUG] Print Release Details" - run: | - echo steps.check_release_commit.outputs.id: ${{ steps.check_release_commit.outputs.id }} - echo steps.check_release_commit.outputs.tag_name: ${{ steps.check_release_commit.outputs.tag_name }} - echo steps.check_release_commit.outputs.target_commitish: ${{ steps.check_release_commit.outputs.target_commitish }} - echo steps.check_release_commit.outputs.prerelease: ${{ steps.check_release_commit.outputs.prerelease }} - - # Since we already know a release for this tag does not exist, if we find anything it's for the wrong tag, exit - - name: "Check If The Tag Matches The Version Number" - if: steps.check_release_commit.outputs.id != '' - run: | - title="Tag ${{ steps.check_release_commit.outputs.tag_name }} already exists for this commit!" - message="Cannot create a new tag for ${{ needs.check-release-exists.outputs.tag }} for the same commit" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - - publish-draft-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: >- - needs.check-release-exists.outputs.draft_exists == 'true' && - inputs.test_run == false - - steps: - - name: "Publish Draft Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release edit $TAG --draft=false --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - - create-github-release: - runs-on: ubuntu-latest - needs: [check-release-exists, audit-release-different-commit] - if: needs.check-release-exists.outputs.draft_exists == 'false' - - steps: - - name: "Check out repository" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.sha }} - - - name: "Download Artifact ${{ inputs.archive_name }}" - uses: actions/download-artifact@v4 - with: - name: ${{ inputs.archive_name }} - path: dist/ - - - name: "[DEBUG] Display Structure Of Expected Files" - run: | - ls -R .changes - ls -l dist - - - name: "Set Release Type" - id: release_type - run: | - if ${{ contains(inputs.version_number, 'rc') || contains(inputs.version_number, 'b') }} - then - echo Release will be set as pre-release - echo "prerelease=--prerelease" >> $GITHUB_OUTPUT - else - echo This is not a prerelease - fi - - - name: "Set As Draft Release" - id: draft - run: | - if [[ ${{ inputs.test_run }} == true ]] - then - echo Release will be published as draft - echo "draft=--draft" >> $GITHUB_OUTPUT - else - echo This is not a draft release - fi - - - name: "GitHub Release Workflow Annotation" - run: | - title="Release ${{ needs.check-release-exists.outputs.tag }}" - message="Configuration: ${{ steps.release_type.outputs.prerelease }} ${{ steps.draft.outputs.draft }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - - name: "Create New GitHub Release - ${{ needs.check-release-exists.outputs.tag }}" - run: | - gh release create $TAG ./dist/* --title "$TITLE" --notes-file $RELEASE_NOTES --target $COMMIT $PRERELEASE $DRAFT --repo ${{ env.REPO_LINK }} - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - TAG: ${{ needs.check-release-exists.outputs.tag }} - TITLE: ${{ github.event.repository.name }} ${{ needs.check-release-exists.outputs.tag }} - RELEASE_NOTES: ${{ inputs.changelog_path }} - COMMIT: ${{ inputs.sha }} - PRERELEASE: ${{ steps.release_type.outputs.prerelease }} - DRAFT: ${{ steps.draft.outputs.draft }} diff --git a/.github/workflows/precommit-autoupdate.yml b/.github/workflows/precommit-autoupdate.yml deleted file mode 100644 index 74976c48..00000000 --- a/.github/workflows/precommit-autoupdate.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: "Run pre-commit autoupdate" - -on: - schedule: - - cron: "30 1 * * SAT" - workflow_dispatch: - -permissions: - contents: write - -concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }} - cancel-in-progress: true - -jobs: - precommit-autoupdate: - name: "Run pre-commit autoupdate" - uses: dbt-labs/actions/.github/workflows/pre-commit-autoupdate.yml - secrets: - TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} - SLACK_WEBHOOK_PR_URL: ${{ secrets.SLACK_DEV_ADAPTER_PULL_REQUESTS }} - SLACK_WEBHOOK_ALERTS_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} diff --git a/.github/workflows/release_prep_hatch.yml b/.github/workflows/release_prep_hatch.yml deleted file mode 100644 index a6105786..00000000 --- a/.github/workflows/release_prep_hatch.yml +++ /dev/null @@ -1,542 +0,0 @@ -# **what?** -# Perform the version bump, generate the changelog and run tests. -# -# Inputs: -# version_number: The release version number (i.e. 1.0.0b1, 1.2.3rc2, 1.0.0) -# target_branch: The branch that we will release from -# env_setup_script_path: Path to the environment setup script -# deploy_to: If we are deploying to prod or test, if test then release from branch -# nightly_release: Identifier that this is nightly release -# -# Outputs: -# final_sha: The sha that will actually be released. This can differ from the -# input sha if adding a version bump and/or changelog -# changelog_path: Path to the changelog file (ex .changes/1.2.3-rc1.md) -# -# Branching strategy: -# - During execution workflow execution the temp branch will be generated. -# - For normal runs the temp branch will be removed once changes were merged to target branch; -# - For test runs we will keep temp branch and will use it for release; -# Naming strategy: -# - For normal runs: prep-release/${{ inputs.deploy_to}}/${{ inputs.version_number }}_$GITHUB_RUN_ID -# - For nightly releases: prep-release/nightly-release/${{ inputs.version_number }}_$GITHUB_RUN_ID -# -# **why?** -# Reusable and consistent GitHub release process. -# -# **when?** -# Call when ready to kick off a build and release -# -# Validation Checks -# -# 1. Bump the version if it has not been bumped -# 2. Generate the changelog (via changie) if there is no markdown file for this version -# - -name: Version Bump and Changelog Generation -run-name: Bump to ${{ inputs.version_number }} for release to ${{ inputs.deploy_to }} and generate changelog -on: - workflow_call: - inputs: - version_number: - required: true - type: string - deploy_to: - type: string - default: prod - required: false - nightly_release: - type: boolean - default: false - required: false - env_setup_script_path: - type: string - required: false - default: '' - run-unit-tests: - type: boolean - default: false - run-integration-tests: - type: boolean - default: false - target_branch: - description: "The branch to release from" - type: string - required: false - default: main - working-dir: - description: "The working directory to use for run statements" - type: string - default: "./" - outputs: - changelog_path: - description: The path to the changelog for this version - value: ${{ jobs.audit-changelog.outputs.changelog_path }} - final_sha: - description: The sha that will actually be released - value: ${{ jobs.determine-release-branch.outputs.final_sha }} - secrets: - FISHTOWN_BOT_PAT: - description: "Token to commit/merge changes into branches" - required: true - IT_TEAM_MEMBERSHIP: - description: "Token that can view org level teams" - required: true - -permissions: - contents: write - -defaults: - run: - shell: bash - -env: - PYTHON_TARGET_VERSION: 3.11 - NOTIFICATION_PREFIX: "[Release Preparation]" - -jobs: - log-inputs: - runs-on: ubuntu-latest - - steps: - - name: "[DEBUG] Print Variables" - run: | - # WORKFLOW INPUTS - echo The release version number: ${{ inputs.version_number }} - echo Deploy to: ${{ inputs.deploy_to }} - echo Target branch: ${{ inputs.target_branch }} - echo Nightly release: ${{ inputs.nightly_release }} - echo Optional env setup script: ${{ inputs.env_setup_script_path }} - echo run-unit-tests: ${{ inputs.run-unit-tests }} - echo run-integration-tests: ${{ inputs.run-integration-tests }} - echo working-dir: ${{ inputs.working-dir }} - # ENVIRONMENT VARIABLES - echo Python target version: ${{ env.PYTHON_TARGET_VERSION }} - echo Notification prefix: ${{ env.NOTIFICATION_PREFIX }} - audit-changelog: - runs-on: ubuntu-latest - - outputs: - changelog_path: ${{ steps.set_path.outputs.changelog_path }} - exists: ${{ steps.set_existence.outputs.exists }} - base_version: ${{ steps.semver.outputs.base-version }} - prerelease: ${{ steps.semver.outputs.pre-release }} - is_prerelease: ${{ steps.semver.outputs.is-pre-release }} - - steps: - - name: "Checkout ${{ github.repository }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: "Audit Version And Parse Into Parts" - id: semver - uses: dbt-labs/actions/parse-semver@v1.1.1 - with: - version: ${{ inputs.version_number }} - - - name: "Set Changelog Path" - id: set_path - run: | - path=".changes/" - if [[ ${{ steps.semver.outputs.is-pre-release }} -eq 1 ]] - then - path+="${{ steps.semver.outputs.base-version }}-${{ steps.semver.outputs.pre-release }}.md" - else - path+="${{ steps.semver.outputs.base-version }}.md" - fi - # Send notification - echo "changelog_path=$path" >> $GITHUB_OUTPUT - title="Changelog path" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$changelog_path" - - name: "Set Changelog Existence For Subsequent Jobs" - id: set_existence - run: | - does_exist=false - if test -f ${{ steps.set_path.outputs.changelog_path }} - then - does_exist=true - fi - echo "exists=$does_exist">> $GITHUB_OUTPUT - - name: "[Notification] Set Changelog Existence For Subsequent Jobs" - run: | - title="Changelog exists" - if [[ ${{ steps.set_existence.outputs.exists }} == true ]] - then - message="Changelog file ${{ steps.set_path.outputs.changelog_path }} already exists" - else - message="Changelog file ${{ steps.set_path.outputs.changelog_path }} doesn't exist" - fi - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo changelog_path: ${{ steps.set_path.outputs.changelog_path }} - echo exists: ${{ steps.set_existence.outputs.exists }} - echo base_version: ${{ steps.semver.outputs.base-version }} - echo prerelease: ${{ steps.semver.outputs.pre-release }} - echo is_prerelease: ${{ steps.semver.outputs.is-pre-release }} - - audit-version-in-code: - runs-on: ubuntu-latest - - outputs: - up_to_date: ${{ steps.version-check.outputs.up_to_date }} - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ inputs.target_branch }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - - name: "Check Current Version In Code" - id: version-check - run: | - is_updated=false - current_version=$(hatch version) - if test "$current_version" = "${{ inputs.version_number }}" - then - is_updated=true - fi - echo "up_to_date=$is_updated" >> $GITHUB_OUTPUT - working-directory: ${{ inputs.working-dir }} - - - name: "[Notification] Check Current Version In Code" - run: | - title="Version check" - if [[ ${{ steps.version-check.outputs.up_to_date }} == true ]] - then - message="The version in the codebase is equal to the provided version" - else - message="The version in the codebase differs from the provided version" - fi - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo up_to_date: ${{ steps.version-check.outputs.up_to_date }} - - skip-generate-changelog: - runs-on: ubuntu-latest - needs: [audit-changelog] - if: needs.audit-changelog.outputs.exists == 'true' - - steps: - - name: "Changelog Exists, Skip Generating New Changelog" - run: | - # Send notification - title="Skip changelog generation" - message="A changelog file already exists at ${{ needs.audit-changelog.outputs.changelog_path }}, skipping generating changelog" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - skip-version-bump: - runs-on: ubuntu-latest - needs: [audit-version-in-code] - if: needs.audit-version-in-code.outputs.up_to_date == 'true' - - steps: - - name: "Version Already Bumped" - run: | - # Send notification - title="Skip version bump" - message="The version has already been bumped to ${{ inputs.version_number }}, skipping version bump" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - create-temp-branch: - runs-on: ubuntu-latest - needs: [audit-changelog, audit-version-in-code] - if: needs.audit-changelog.outputs.exists == 'false' || needs.audit-version-in-code.outputs.up_to_date == 'false' - - outputs: - branch_name: ${{ steps.variables.outputs.branch_name }} - - steps: - - name: "Checkout ${{ github.repository }}" - uses: actions/checkout@v4 - with: - ref: ${{ inputs.target_branch }} - - - name: "Generate Branch Name" - id: variables - run: | - name="prep-release/" - if [[ ${{ inputs.nightly_release }} == true ]] - then - name+="nightly-release/" - else - name+="${{ inputs.deploy_to }}/" - fi - name+="${{ inputs.version_number }}_$GITHUB_RUN_ID" - echo "branch_name=$name" >> $GITHUB_OUTPUT - - name: "Create Branch - ${{ steps.variables.outputs.branch_name }}" - run: | - git checkout -b ${{ steps.variables.outputs.branch_name }} - git push -u origin ${{ steps.variables.outputs.branch_name }} - - name: "[Notification] Temp branch created" - run: | - # Send notification - title="Temp branch generated" - message="The ${{ steps.variables.outputs.branch_name }} branch created" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "[DEBUG] Print Outputs" - run: | - echo branch_name ${{ steps.variables.outputs.branch_name }} - generate-changelog-bump-version: - runs-on: ubuntu-latest - needs: [audit-changelog, audit-version-in-code, create-temp-branch] - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v4 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - name: Setup `hatch` - uses: ./.github/actions/setup-hatch - - name: "Add Homebrew To PATH" - run: | - echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH - - name: "Install Homebrew Packages" - run: | - brew install pre-commit - brew tap miniscruff/changie https://github.com/miniscruff/changie - brew install changie - - name: "Set json File Name" - id: json_file - run: | - echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT - - name: "Get Core Team Membership" - run: | - gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.json_file.outputs.name }} - env: - GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} - - name: "Set Core Team Membership for Changie Contributors exclusion" - id: set_team_membership - run: | - team_list=$(jq -r '.[].login' ${{ steps.json_file.outputs.name }}) - echo $team_list - team_list_single=$(echo $team_list | tr '\n' ' ') - echo "CHANGIE_CORE_TEAM=$team_list_single" >> $GITHUB_ENV - - name: "Delete the json File" - run: | - rm ${{ steps.json_file.outputs.name }} - - name: "Generate Release Changelog" - if: needs.audit-changelog.outputs.exists == 'false' - run: | - if [[ ${{ needs.audit-changelog.outputs.is_prerelease }} -eq 1 ]] - then - changie batch ${{ needs.audit-changelog.outputs.base_version }} --move-dir '${{ needs.audit-changelog.outputs.base_version }}' --prerelease ${{ needs.audit-changelog.outputs.prerelease }} - elif [[ -d ".changes/${{ needs.audit-changelog.outputs.base_version }}" ]] - then - changie batch ${{ needs.audit-changelog.outputs.base_version }} --include '${{ needs.audit-changelog.outputs.base_version }}' --remove-prereleases - else # releasing a final patch with no prereleases - changie batch ${{ needs.audit-changelog.outputs.base_version }} - fi - changie merge - git status - - name: "Check Changelog Created Successfully" - if: needs.audit-changelog.outputs.exists == 'false' - run: | - title="Changelog" - if [[ -f ${{ needs.audit-changelog.outputs.changelog_path }} ]] - then - message="Changelog file created successfully" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - else - message="Changelog failed to generate" - echo "::error title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - exit 1 - fi - - name: "Bump Version To ${{ inputs.version_number }}" - if: needs.audit-version-in-code.outputs.up_to_date == 'false' - run: | - hatch version ${{ inputs.version_number }} - working-directory: ${{ inputs.working-dir }} - - name: "[Notification] Bump Version To ${{ inputs.version_number }}" - if: needs.audit-version-in-code.outputs.up_to_date == 'false' - run: | - title="Version bump" - message="Version successfully bumped in codebase to ${{ inputs.version_number }}" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - # TODO: can these 2 steps be done via hatch? probably. - # this step will fail on whitespace errors but also correct them - - name: "Remove Trailing Whitespace Via Pre-commit" - continue-on-error: true - run: | - pre-commit run trailing-whitespace --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - git status - # this step will fail on newline errors but also correct them - - name: "Removing Extra Newlines Via Pre-commit" - continue-on-error: true - run: | - pre-commit run end-of-file-fixer --files dbt/adapters/__about__.py CHANGELOG.md .changes/* - git status - - name: "Commit & Push Changes" - run: | - #Data for commit - user="Github Build Bot" - email="buildbot@fishtownanalytics.com" - commit_message="Bumping version to ${{ inputs.version_number }} and generate changelog" - #Commit changes to branch - git config user.name "$user" - git config user.email "$email" - git pull - git add . - git commit -m "$commit_message" - git push - - run-unit-tests: - if: inputs.run-unit-tests == true - runs-on: ubuntu-latest - needs: [create-temp-branch, generate-changelog-bump-version] - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v4 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - name: "Setup `hatch`" - uses: ./.github/actions/setup-hatch - - name: "Run Unit Tests" - run: hatch run unit-tests - - run-integration-tests: - runs-on: ubuntu-20.04 - needs: [create-temp-branch, generate-changelog-bump-version] - if: inputs.run-integration-tests == true - - steps: - - name: "Checkout ${{ github.repository }} Branch ${{ needs.create-temp-branch.outputs.branch_name }}" - uses: actions/checkout@v4 - with: - ref: ${{ needs.create-temp-branch.outputs.branch_name }} - - - name: "Setup Environment Variables - ./${{ inputs.env_setup_script_path }}" - if: inputs.env_setup_script_path != '' - run: source ./${{ inputs.env_setup_script_path }} - - - name: "Setup Environment Variables - Secrets Context" - if: inputs.env_setup_script_path != '' - uses: actions/github-script@v6 - id: check-env - with: - result-encoding: string - script: | - try { - const { SECRETS_CONTEXT, INTEGRATION_TESTS_SECRETS_PREFIX } = process.env - const secrets = JSON.parse(SECRETS_CONTEXT) - if (INTEGRATION_TESTS_SECRETS_PREFIX) { - for (const [key, value] of Object.entries(secrets)) { - if (key.startsWith(INTEGRATION_TESTS_SECRETS_PREFIX)) { - core.exportVariable(key, value) - } - } - } else { - core.info("The INTEGRATION_TESTS_SECRETS_PREFIX env variable is empty or not defined, skipping the secrets setup.") - } - } catch (err) { - core.error("Error while reading or parsing the JSON") - core.setFailed(err) - } - env: - SECRETS_CONTEXT: ${{ toJson(secrets) }} - - - name: "Set up Python & Hatch - ${{ env.PYTHON_TARGET_VERSION }}" - uses: ./.github/actions/setup-python-env - with: - python-version: ${{ env.PYTHON_TARGET_VERSION }} - - - name: Run tests - run: hatch run integration-tests - - merge-changes-into-target-branch: - runs-on: ubuntu-latest - needs: [run-unit-tests, run-integration-tests, create-temp-branch, audit-version-in-code, audit-changelog] - if: | - !failure() && !cancelled() && - inputs.deploy_to == 'prod' && - ( - needs.audit-changelog.outputs.exists == 'false' || - needs.audit-version-in-code.outputs.up_to_date == 'false' - ) - steps: - - name: "[Debug] Print Variables" - run: | - echo branch_name: ${{ needs.create-temp-branch.outputs.branch_name }} - echo inputs.deploy_to: ${{ inputs.deploy_to }} - echo needs.audit-changelog.outputs.exists: ${{ needs.audit-changelog.outputs.exists }} - echo needs.audit-version-in-code.outputs.up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - - name: "Checkout Repo ${{ github.repository }}" - uses: actions/checkout@v4 - - - name: "Merge Changes Into ${{ inputs.target_branch }}" - uses: everlytic/branch-merge@1.1.5 - with: - source_ref: ${{ needs.create-temp-branch.outputs.branch_name }} - target_branch: ${{ inputs.target_branch }} - github_token: ${{ secrets.FISHTOWN_BOT_PAT }} - commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" - - - name: "[Notification] Changes Merged into main" - run: | - title="Changelog and Version Bump Branch Merge" - message="The ${{ needs.create-temp-branch.outputs.branch_name }} branch was merged into mains" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - determine-release-branch: - runs-on: ubuntu-latest - needs: - [ - create-temp-branch, - merge-changes-into-target-branch, - audit-changelog, - audit-version-in-code, - ] - # always run this job, regardless of if the dependant jobs were skipped - if: ${{ !failure() && !cancelled() }} - - # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, - # then it is what we will release. Otherwise we generated a changelog and did the version bump in this workflow and there is a - # new sha to use from the merge we just did. Grab that here instead. - outputs: - final_sha: ${{ steps.resolve_commit_sha.outputs.release_sha }} - - steps: - - name: "[Debug] Print Variables" - run: | - echo new_branch: ${{ needs.create-temp-branch.outputs.branch_name }} - echo changelog_exists: ${{ needs.audit-changelog.outputs.exists }} - echo up_to_date: ${{ needs.audit-version-in-code.outputs.up_to_date }} - - name: "Resolve Branch To Checkout" - id: resolve_branch - run: | - branch="" - if [ ${{ inputs.deploy_to == 'test' }}] || [ ${{ inputs.nightly_release == 'true' }} ] - then - branch=${{ needs.create-temp-branch.outputs.branch_name }} - else - branch="${{ inputs.target_branch }}" - fi - echo "target_branch=$branch" >> $GITHUB_OUTPUT - - name: "[Notification] Resolve Branch To Checkout" - run: | - title="Branch pick" - message="The ${{ steps.resolve_branch.outputs.target_branch }} branch will be used for release" - echo "::notice title=${{ env.NOTIFICATION_PREFIX }}: $title::$message" - - name: "Checkout Resolved Branch - ${{ steps.resolve_branch.outputs.target_branch }}" - uses: actions/checkout@v4 - with: - ref: ${{ steps.resolve_branch.outputs.target_branch }} - - - name: "[Debug] Log Branch" - run: git status - - - name: "Resolve Commit SHA For Release" - id: resolve_commit_sha - run: | - echo "release_sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT - - - name: "Remove Temp Branch - ${{ needs.create-temp-branch.outputs.branch_name }}" - if: ${{ inputs.deploy_to == 'prod' && inputs.nightly_release == 'false' && needs.create-temp-branch.outputs.branch_name != '' }} - run: | - git push origin -d ${{ needs.create-temp-branch.outputs.branch_name }} From ceadd71df6ea7bf9b0be6d78c400c1e293135a38 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 17:45:58 -0500 Subject: [PATCH 21/63] update publish to work with dbt-adapters --- .github/workflows/_generate-changelog.yml | 229 ++++++++++++++++++++++ .github/workflows/_package-directory.yml | 26 +++ .github/workflows/_publish-internal.yml | 20 +- .github/workflows/_publish-pypi.yml | 12 +- .github/workflows/publish.yml | 14 +- 5 files changed, 287 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/_generate-changelog.yml create mode 100644 .github/workflows/_package-directory.yml diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml new file mode 100644 index 00000000..2b52864f --- /dev/null +++ b/.github/workflows/_generate-changelog.yml @@ -0,0 +1,229 @@ +name: "Changelog generation" + +on: + workflow_call: + inputs: + package: + description: "Choose the package getting published" + type: string + default: "dbt-adapters" + deploy-to: + description: "Choose whether we're publishing to test or prod" + type: string + default: "prod" + branch: + description: "Choose the branch to use" + type: string + default: "main" + outputs: + branch-name: + description: "The SHA to release" + value: ${{ jobs.branch.outputs.name }} + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + workflow_dispatch: + inputs: + package: + description: "Choose the package getting published" + type: string + default: "dbt-adapters" + deploy-to: + description: "Choose whether we're publishing to test or prod" + type: environment + default: "prod" + branch: + description: "Choose the branch to use" + type: string + default: "main" + secrets: + FISHTOWN_BOT_PAT: + description: "Token to commit/merge changes into branches" + required: true + IT_TEAM_MEMBERSHIP: + description: "Token that can view org level teams" + required: true + +permissions: + contents: write + +defaults: + run: + shell: bash + +jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + + version: + needs: package + runs-on: ${{ vars.DEFAULT_RUNNER }} + outputs: + raw: ${{ steps.version.outputs.raw }} + base: ${{ steps.semver.outputs.base-version }} + prerelease: ${{ steps.semver.outputs.pre-release }} + is-prerelease: ${{ steps.semver.outputs.is-pre-release }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ vars.DEFAULT_PYTHON_VERSION }} + - uses: pypa/hatch@install + - id: version + run: echo "raw=$(hatch version)" >> $GITHUB_OUTPUT + working-directory: ./${{ needs.package.outputs.directory }} + - id: semver + uses: dbt-labs/actions/parse-semver@v1.1.1 + with: + version: ${{ steps.version.outputs.raw }} + + changelog: + runs-on: ${{ vars.DEFAULT_RUNNER }} + needs: [package, version] + outputs: + path: ${{ steps.changelog.outputs.path }} + exists: ${{ steps.changelog.outputs.exists }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - id: changelog + run: | + path=".changes/${{ needs.version.outputs.base }}" + if [[ ${{ needs.version.outputs.is-prerelease }} -eq 1 ]] + then + path+="-${{ needs.version.outputs.prerelease }}" + fi + path+=".md" + + echo "path=$path" >> $GITHUB_OUTPUT + + exists=false + if test -f $path + then + exists=true + fi + echo "exists=$exists">> $GITHUB_OUTPUT + working-directory: ./${{ needs.package.outputs.directory }} + + temp-branch: + needs: [version, changelog] + if: needs.changelog.outputs.exists == 'false' + runs-on: ${{ vars.DEFAULT_RUNNER }} + outputs: + name: ${{ steps.branch.outputs.name }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - id: branch + run: | + name="prep-release/${{ inputs.package }}/${{ inputs.deploy_to }}/${{ needs.version.outputs.raw }}_$GITHUB_RUN_ID" + echo "name=$name" >> $GITHUB_OUTPUT + - run: | + git checkout -b ${{ steps.branch.outputs.name }} + git push -u origin ${{ steps.branch.outputs.name }} + + dbt-membership: + needs: [version, changelog] + if: needs.changelog.outputs.exists == 'false' + runs-on: ${{ vars.DEFAULT_RUNNER }} + outputs: + team: ${{ steps.team.outputs.team }} + steps: + - id: temp-file + run: echo "name=output_$GITHUB_RUN_ID.json" >> $GITHUB_OUTPUT + - run: | + gh api -H "Accept: application/vnd.github+json" orgs/dbt-labs/teams/core-group/members > ${{ steps.temp-file.outputs.name }} + env: + GH_TOKEN: ${{ secrets.IT_TEAM_MEMBERSHIP }} + - id: team + run: | + team_list=$(jq -r '.[].login' ${{ steps.temp-file.outputs.name }}) + team_list_single=$(echo $team_list | tr '\n' ' ') + echo "team=$team_list_single" >> $GITHUB_OUTPUT + - run: rm ${{ steps.temp-file.outputs.name }} + + generate-changelog: + needs: [package, version, changelog, temp-branch, dbt-membership] + if: needs.changelog.outputs.exists == 'false' + runs-on: ${{ vars.DEFAULT_RUNNER }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.temp-branch.outputs.name }} + - run: echo "/home/linuxbrew/.linuxbrew/bin:/home/linuxbrew/.linuxbrew/sbin" >> $GITHUB_PATH + - run: | + brew install pre-commit + brew tap miniscruff/changie https://github.com/miniscruff/changie + brew install changie + - run: | + if [[ ${{ needs.version.outputs.is-prerelease }} -eq 1 ]] + then + changie batch ${{ needs.version.outputs.base }} --move-dir '${{ needs.version.outputs.base }}' --prerelease ${{ needs.version.outputs.prerelease }} + elif [[ -d ".changes/${{ needs.version.outputs.base }}" ]] + then + changie batch ${{ needs.version.outputs.base }} --include '${{ needs.version.outputs.base }}' --remove-prereleases + else # releasing a final patch with no prereleases + changie batch ${{ needs.version.outputs.base }} + fi + changie merge + working-directory: ./${{ needs.package.outputs.directory }} + env: + CHANGIE_CORE_TEAM: ${{ needs.dbt-membership.outputs.team }} + - run: | + pre-commit run trailing-whitespace --files __version__.py CHANGELOG.md .changes/* + pre-commit run end-of-file-fixer --files __version__.py CHANGELOG.md .changes/* + working-directory: ./${{ needs.package.outputs.directory }} + continue-on-error: true + - run: | + git config user.name "Github Build Bot" + git config user.email "buildbot@fishtownanalytics.com" + git pull + git add . + git commit -m "generate changelog" + git push + working-directory: ./${{ needs.package.outputs.directory }} + + merge-changes: + runs-on: ${{ vars.DEFAULT_RUNNER }} + needs: [temp-branch, generate-changelog] + if: ${{ needs.temp-branch.outputs.name != '' && inputs.deploy-to == 'prod' }} + steps: + - uses: everlytic/branch-merge@1.1.5 + with: + source_ref: ${{ needs.temp-branch.outputs.name }} + target_branch: ${{ inputs.branch }} + github_token: ${{ secrets.FISHTOWN_BOT_PAT }} + commit_message_template: "[Automated] Merged {source_ref} into target {target_branch} during release process" + - run: git push origin -d ${{ needs.temp-branch.outputs.name }} + + branch: + needs: [temp-branch, merge-changes] + if: ${{ !failure() && !cancelled() }} + runs-on: ${{ vars.DEFAULT_RUNNER }} + # always run this job, regardless of whether changelog generation was skipped + # Get the sha that will be released. If the changelog already exists on the input sha and the version has already been bumped, + # then it is what we will release. Otherwise, we generated a changelog and did the version bump in this workflow and there is a + # new sha to use from the merge we just did. Grab that here instead. + outputs: + name: ${{ steps.branch.outputs.name }} + steps: + - id: branch + run: | + branch="" + if [ ${{ needs.temp-branch.outputs.name == '' || inputs.deploy-to == 'prod' }}] + then + branch="${{ inputs.branch }}" + else + branch=${{ needs.temp-branch.outputs.name }} + fi + echo "name=$branch" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml new file mode 100644 index 00000000..11c43b3a --- /dev/null +++ b/.github/workflows/_package-directory.yml @@ -0,0 +1,26 @@ +name: "Package directory" + +on: + workflow_call: + inputs: + package: + description: "Choose the package whose directory you need" + type: string + default: "dbt-adapters" + +jobs: + package: + runs-on: ${{ vars.DEFAULT_RUNNER }} + outputs: + directory: ${{ steps.package.outputs.directory }} + steps: + - id: package + shell: bash + runs: | + if [[ package == 'dbt-adapters' ]] + then + directory="" + else + directory="${{ inputs.package }}" + fi + echo "directory=$directory" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml index 815c0f70..1c339701 100644 --- a/.github/workflows/_publish-internal.yml +++ b/.github/workflows/_publish-internal.yml @@ -35,7 +35,13 @@ defaults: shell: bash jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + publish: + needs: package runs-on: ${{ vars.DEFAULT_RUNNER }} environment: name: ${{ inputs.deploy-to }} @@ -57,7 +63,7 @@ jobs: # strip the pre-release off to find all iterations of this patch hatch version release echo "version=$(hatch version)" >> $GITHUB_OUTPUT - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} - id: published run: | versions_published="$(aws codeartifact list-package-versions \ @@ -73,14 +79,12 @@ jobs: with: version_number: ${{ steps.package.outputs.version }} versions_published: ${{ steps.published.outputs.versions }} - - name: "Update version to internal PyPI format" - run: | + - run: | VERSION=${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD) tee <<< "version = \"$VERSION\"" ./src/dbt/adapters/athena/__version__.py - working-directory: ./${{ inputs.package }} - - name: "Remove dbt-core from build requirements" - run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} + - run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml + working-directory: ./${{ needs.package.outputs.directory }} - run: | export HATCH_INDEX_USER=${{ secrets.AWS_USER }} @@ -99,4 +103,4 @@ jobs: hatch build --clean hatch run build:check-all hatch publish - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml index d8eed4c1..4150631d 100644 --- a/.github/workflows/_publish-pypi.yml +++ b/.github/workflows/_publish-pypi.yml @@ -38,7 +38,13 @@ defaults: shell: bash jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + publish: + needs: package runs-on: ${{ vars.DEFAULT_RUNNER }} environment: name: ${{ inputs.deploy-to }} @@ -58,7 +64,7 @@ jobs: # hatch will build using test PyPI first and fall back to prod PyPI when deploying to test # this is done via environment variables in the test environment in GitHub - run: hatch build && hatch run build:check-all - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} - uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: ${{ vars.PYPI_REPOSITORY_URL }} @@ -66,7 +72,7 @@ jobs: verify: runs-on: ${{ vars.DEFAULT_RUNNER }} - needs: publish + needs: [package, publish] # check the correct index environment: name: ${{ inputs.deploy-to }} @@ -76,7 +82,7 @@ jobs: ref: ${{ inputs.branch }} - id: version run: echo "version=$(hatch version)" >> $GITHUB_OUTPUT - working-directory: ./${{ inputs.package }} + working-directory: ./${{ needs.package.outputs.directory }} - uses: nick-fields/retry@v3 with: timeout_seconds: 10 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 31558056..9a158097 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -38,14 +38,22 @@ jobs: package: ${{ inputs.package }} branch: ${{ inputs.branch }} + generate-changelog: + uses: ./.github/workflows/_generate-changelog.yml + with: + package: ${{ inputs.package }} + deploy-to: ${{ inputs.deploy-to }} + branch: ${{ inputs.branch }} + secrets: inherit + publish-internal: if: ${{ inputs.pypi-internal == true }} - needs: unit-tests + needs: [unit-tests, generate-changelog] uses: ./.github/workflows/_publish-internal.yml with: package: ${{ inputs.package }} deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} + branch: ${{ needs.generate-changelog.outputs.branch-name }} secrets: inherit publish-pypi: @@ -55,4 +63,4 @@ jobs: with: package: ${{ inputs.package }} deploy-to: ${{ inputs.deploy-to }} - branch: ${{ inputs.branch }} + branch: ${{ needs.generate-changelog.outputs.branch-name }} From e57b1b240d0bcfdac1f90bd16e024bbbb5e4e423 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 17:46:17 -0500 Subject: [PATCH 22/63] clean up existing workflows --- .github/workflows/changelog-exists.yml | 21 ++++++++ .github/workflows/issue-triage.yml | 34 +++++-------- .github/workflows/scheduled-maintenance.yml | 42 +++++++--------- .github/workflows/user-docs.yml | 54 +++++++++------------ 4 files changed, 72 insertions(+), 79 deletions(-) create mode 100644 .github/workflows/changelog-exists.yml diff --git a/.github/workflows/changelog-exists.yml b/.github/workflows/changelog-exists.yml new file mode 100644 index 00000000..1b5a2423 --- /dev/null +++ b/.github/workflows/changelog-exists.yml @@ -0,0 +1,21 @@ +name: "Check changelog entry exists" + +on: + pull_request_target: + types: [opened, reopened, labeled, unlabeled, synchronize] + +defaults: + run: + shell: bash + +permissions: + contents: read + pull-requests: write + +jobs: + changelog: + uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main + with: + changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' + skip_label: 'Skip Changelog' + secrets: inherit diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 385ef820..5432121f 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -1,31 +1,19 @@ -# **what?** -# When triaging submissions, we sometimes need more information from the issue creator. -# In those cases we remove the `triage` label and add the `awaiting_response` label. -# Once we receive a response in the form of a comment, we want the `awaiting_response` label removed -# and the `triage` label added so that we are aware that the issue needs action. - -# **why?** -# This automates a part of issue triaging while also removing noise from triage lists. - -# **when?** -# This will run when a comment is added to an issue and that issue has an `awaiting_response` label. - -name: Resubmit for Triage +name: "Issue triage" on: issue_comment defaults: - run: - shell: bash + run: + shell: bash permissions: - issues: write + issues: write jobs: - triage_label: - if: contains(github.event.issue.labels.*.name, 'awaiting_response') - uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main - with: - add_label: "triage" - remove_label: "awaiting_response" - secrets: inherit # this is only acceptable because we own the action we're calling + triage: + if: contains(github.event.issue.labels.*.name, 'triage:awaiting-response') + uses: dbt-labs/actions/.github/workflows/swap-labels.yml@main + with: + add_label: "triage:dbt" + remove_label: "triage:awaiting-response" + secrets: inherit diff --git a/.github/workflows/scheduled-maintenance.yml b/.github/workflows/scheduled-maintenance.yml index 75a14dd4..28309b95 100644 --- a/.github/workflows/scheduled-maintenance.yml +++ b/.github/workflows/scheduled-maintenance.yml @@ -1,30 +1,24 @@ -# **what?** -# For issues that have been open for awhile without activity, label -# them as stale with a warning that they will be closed out. If -# anyone comments to keep the issue open, it will automatically -# remove the stale label and keep it open. +name: "Scheduled maintenance" -# Stale label rules: -# awaiting_response, more_information_needed -> 90 days -# good_first_issue, help_wanted -> 360 days (a year) -# tech_debt -> 720 (2 years) -# all else defaults -> 180 days (6 months) - -# **why?** -# To keep the repo in a clean state from issues that aren't relevant anymore - -# **when?** -# Once a day - -name: "Close stale issues and PRs" on: - schedule: - - cron: "30 1 * * *" + schedule: + - cron: "30 1 * * SAT" + workflow_dispatch: permissions: - issues: write - pull-requests: write + contents: write + +concurrency: + group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }} + cancel-in-progress: true jobs: - stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main + pre-commit-autoupdate: + uses: dbt-labs/actions/.github/workflows/pre-commit-autoupdate.yml + secrets: + TOKEN: ${{ secrets.FISHTOWN_BOT_PAT }} + SLACK_WEBHOOK_PR_URL: ${{ secrets.SLACK_DEV_ADAPTER_PULL_REQUESTS }} + SLACK_WEBHOOK_ALERTS_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} + + stale: + uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main diff --git a/.github/workflows/user-docs.yml b/.github/workflows/user-docs.yml index f49cf517..66bd3da1 100644 --- a/.github/workflows/user-docs.yml +++ b/.github/workflows/user-docs.yml @@ -1,41 +1,31 @@ -# **what?** -# Open an issue in docs.getdbt.com when an issue is labeled `user docs` and closed as completed - -# **why?** -# To reduce barriers for keeping docs up to date - -# **when?** -# When an issue is labeled `user docs` and is closed as completed. Can be labeled before or after the issue is closed. - - -name: Open issues in docs.getdbt.com repo when an issue is labeled -run-name: "Open an issue in docs.getdbt.com for issue #${{ github.event.issue.number }}" +name: "Open user docs issue" on: - issues: - types: [labeled, closed] + issues: + types: [labeled, closed] defaults: - run: - shell: bash + run: + shell: bash permissions: issues: write # comments on issues jobs: - open_issues: - # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. - # If this logic does not exist in this workflow, it runs the - # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having - # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we - # decide if it should run or not. - if: | - (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( - (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || - (github.event.action == 'labeled' && github.event.label.name == 'user docs')) - uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main - with: - issue_repository: "dbt-labs/docs.getdbt.com" - issue_title: "Docs Changes Needed from ${{ github.event.repository.name }} Issue #${{ github.event.issue.number }}" - issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." - secrets: inherit + open_issues: + # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. + # If this logic does not exist in this workflow, it runs the + # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having + # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we + # decide if it should run or not. + if: | + (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( + (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || + (github.event.action == 'labeled' && github.event.label.name == 'user docs') + ) + uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main + with: + issue_repository: "dbt-labs/docs.getdbt.com" + issue_title: "Docs Changes Needed from dbt-adapters - Issue #${{ github.event.issue.number }}" + issue_body: "At a minimum, update body to include a link to the page on docs.getdbt.com requiring updates and what part(s) of the page you would like to see updated." + secrets: inherit From 44fa2e0f296d82a79e97ffe9f1768ada12aec665 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 17:46:40 -0500 Subject: [PATCH 23/63] update pull-request-checks to work with dbt-adapters --- .github/workflows/_unit-tests.yml | 1 - .github/workflows/_verify-build.yml | 73 +++++++++++++++++++++++ .github/workflows/pull-request-checks.yml | 20 +++++-- 3 files changed, 89 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/_verify-build.yml diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index be2cac09..8e113b88 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -63,4 +63,3 @@ jobs: - uses: pypa/hatch@install - run: hatch run unit-tests shell: bash - working-directory: ./${{ inputs.package }} diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml new file mode 100644 index 00000000..241b28c0 --- /dev/null +++ b/.github/workflows/_verify-build.yml @@ -0,0 +1,73 @@ +name: "Verify build" + +on: + workflow_call: + inputs: + package: + description: "Choose the package to build" + type: string + default: "dbt-adapters" + branch: + description: "Choose the branch to build" + type: string + default: "main" + repository: + description: "Choose the repository to build, (used primarily when testing a fork)" + type: string + default: "dbt-labs/dbt-adapters" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: string + default: ${{ vars.DEFAULT_PYTHON_VERSION }} + workflow_dispatch: + inputs: + package: + description: "Choose the package to build" + type: choice + options: ["dbt-adapters", "dbt-tests-adapter"] + branch: + description: "Choose the branch to build" + type: string + default: "main" + repository: + description: "Choose the repository to build, (used primarily when testing a fork)" + type: string + default: "dbt-labs/dbt-adapters" + os: + description: "Choose the OS to test against" + type: string + default: ${{ vars.DEFAULT_RUNNER }} + python-version: + description: "Choose the Python version to test against" + type: choice + options: ["3.9", "3.10", "3.11", "3.12"] + +permissions: read-all + +defaults: + run: + shell: bash + +jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + + build: + needs: package + runs-on: ${{ inputs.os }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.branch }} + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + - uses: pypa/hatch@install + - run: hatch build && hatch run build:check-all + working-directory: ./${{ needs.package.outputs.directory }} diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 2322ff91..770deef3 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -16,21 +16,33 @@ jobs: branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} - unit-tests: - uses: ./.github/workflows/_unit-tests.yml + verify-builds: + uses: ./.github/workflows/_verify-build.yml + strategy: + matrix: + package: ["dbt-adapters", "dbt-tests-adapter"] + python-version: ["3.9", "3.10", "3.11", "3.12"] with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} repository: ${{ github.event.pull_request.head.repo.full_name }} + python-version: ${{ matrix.python-version }} + + unit-tests: + uses: ./.github/workflows/_unit-tests.yml strategy: matrix: package: ["dbt-adapters"] + with: + package: ${{ matrix.package }} + branch: ${{ github.event.pull_request.head.ref }} + repository: ${{ github.event.pull_request.head.repo.full_name }} # This job does nothing and is only used for branch protection results: - name: "Pull request checks" + name: "Pull request checks" # keep this name, branch protection references it if: always() - needs: [code-quality, unit-tests] + needs: [code-quality, verify-builds, unit-tests] runs-on: ${{ vars.DEFAULT_RUNNER }} steps: - uses: re-actors/alls-green@release/v1 From f9972199b5ff8d428284b0cfd0a7cef03a672ef8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:12:28 -0500 Subject: [PATCH 24/63] add pull_request target for checking workflows --- .github/workflows/pull-request-checks.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 770deef3..ded0f443 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,6 +1,8 @@ name: "Pull request checks" on: + pull_request: + types: [opened, reopened, synchronize] pull_request_target: types: [opened, reopened, synchronize] From c3dc37fb2bcbf7bf2102f0ea973c11af772245f6 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:16:49 -0500 Subject: [PATCH 25/63] fix typo in _package-directory.yml --- .github/workflows/_package-directory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 11c43b3a..115abdfe 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -16,7 +16,7 @@ jobs: steps: - id: package shell: bash - runs: | + run: | if [[ package == 'dbt-adapters' ]] then directory="" From f7162c131f63ec0fd456ab0c6bbde3bde332093b Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:23:22 -0500 Subject: [PATCH 26/63] remove env vars from inputs --- .github/workflows/_unit-tests.yml | 6 +++--- .github/workflows/_verify-build.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index 8e113b88..7e6b40f8 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -18,11 +18,11 @@ on: os: description: "Choose the OS to test against" type: string - default: ${{ vars.DEFAULT_RUNNER }} + default: "ubuntu-22.04" python-version: description: "Choose the Python version to test against" type: string - default: ${{ vars.DEFAULT_PYTHON_VERSION }} + default: 3.9 workflow_dispatch: inputs: package: @@ -40,7 +40,7 @@ on: os: description: "Choose the OS to test against" type: string - default: ${{ vars.DEFAULT_RUNNER }} + default: "ubuntu-22.04" python-version: description: "Choose the Python version to test against" type: choice diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml index 241b28c0..a5c33030 100644 --- a/.github/workflows/_verify-build.yml +++ b/.github/workflows/_verify-build.yml @@ -18,11 +18,11 @@ on: os: description: "Choose the OS to test against" type: string - default: ${{ vars.DEFAULT_RUNNER }} + default: "ubuntu-22.04" python-version: description: "Choose the Python version to test against" type: string - default: ${{ vars.DEFAULT_PYTHON_VERSION }} + default: "3.9" workflow_dispatch: inputs: package: @@ -40,7 +40,7 @@ on: os: description: "Choose the OS to test against" type: string - default: ${{ vars.DEFAULT_RUNNER }} + default: "ubuntu-22.04" python-version: description: "Choose the Python version to test against" type: choice From 4be4c46e3ff0031b28e5984075c3bb393bb6b37c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:28:27 -0500 Subject: [PATCH 27/63] add python version to unit test matrix --- .github/workflows/pull-request-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index ded0f443..e6c7de36 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -35,6 +35,7 @@ jobs: strategy: matrix: package: ["dbt-adapters"] + python-version: ["3.9", "3.10", "3.11", "3.12"] with: package: ${{ matrix.package }} branch: ${{ github.event.pull_request.head.ref }} From 2f5917eb3c3204d4b52185c7e22c7f6aef0915ff Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:57:05 -0500 Subject: [PATCH 28/63] register workflow with gha --- .github/workflows/publish.yml | 9 +++++---- pyproject.toml | 1 + 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9a158097..e7d19014 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,6 +1,7 @@ name: "Publish" on: + pull_request: workflow_dispatch: inputs: package: @@ -12,19 +13,19 @@ on: deploy-to: description: "Choose whether to publish to test or prod" type: environment - default: "prod" + default: "test" branch: description: "Choose the branch to publish from" type: string - default: "main" + default: "monorepo/shared-workflows" pypi-internal: description: "Publish Internally" type: boolean - default: true + default: false pypi-public: description: "Publish to PyPI" type: boolean - default: false + default: true # don't attempt to release the same target in parallel concurrency: diff --git a/pyproject.toml b/pyproject.toml index 47cd3ece..36c63556 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,7 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" +test-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" [tool.hatch.envs.build] detached = true From c8ec78a3dfbaf8ae4da341351ad2a62449881075 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 18:58:05 -0500 Subject: [PATCH 29/63] revert defaults --- .github/workflows/publish.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index e7d19014..9a158097 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,7 +1,6 @@ name: "Publish" on: - pull_request: workflow_dispatch: inputs: package: @@ -13,19 +12,19 @@ on: deploy-to: description: "Choose whether to publish to test or prod" type: environment - default: "test" + default: "prod" branch: description: "Choose the branch to publish from" type: string - default: "monorepo/shared-workflows" + default: "main" pypi-internal: description: "Publish Internally" type: boolean - default: false + default: true pypi-public: description: "Publish to PyPI" type: boolean - default: true + default: false # don't attempt to release the same target in parallel concurrency: From 1aeafdfac4a5735c179364643b01d68b5db67d33 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:12:14 -0500 Subject: [PATCH 30/63] update workflow deps for publish.yml --- .github/workflows/publish.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9a158097..5fe98574 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -39,6 +39,7 @@ jobs: branch: ${{ inputs.branch }} generate-changelog: + needs: unit-tests uses: ./.github/workflows/_generate-changelog.yml with: package: ${{ inputs.package }} @@ -48,7 +49,7 @@ jobs: publish-internal: if: ${{ inputs.pypi-internal == true }} - needs: [unit-tests, generate-changelog] + needs: generate-changelog uses: ./.github/workflows/_publish-internal.yml with: package: ${{ inputs.package }} @@ -58,7 +59,7 @@ jobs: publish-pypi: if: ${{ inputs.pypi-public == true }} - needs: unit-tests + needs: generate-changelog uses: ./.github/workflows/_publish-pypi.yml with: package: ${{ inputs.package }} From 5860f1da65a9855961450c9e046e0e4ee7713f44 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:15:52 -0500 Subject: [PATCH 31/63] bump version just to test changelog generation --- dbt/adapters/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 08e0d06b..57ac4b25 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.10.4" +version = "1.10.3a1" From 37716033ea5dac359e281e5e1586946c492eed49 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:22:33 -0500 Subject: [PATCH 32/63] fix temp branch name for changelog generation --- .github/workflows/_generate-changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml index 2b52864f..aca712b2 100644 --- a/.github/workflows/_generate-changelog.yml +++ b/.github/workflows/_generate-changelog.yml @@ -126,7 +126,7 @@ jobs: ref: ${{ inputs.branch }} - id: branch run: | - name="prep-release/${{ inputs.package }}/${{ inputs.deploy_to }}/${{ needs.version.outputs.raw }}_$GITHUB_RUN_ID" + name="prep-release/${{ inputs.package }}/$GITHUB_RUN_ID" echo "name=$name" >> $GITHUB_OUTPUT - run: | git checkout -b ${{ steps.branch.outputs.name }} From 3297a3cce54d32ceaaa2b475f6264fdb96b53d3a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:35:49 -0500 Subject: [PATCH 33/63] revert version bump used for testing publish action --- dbt/adapters/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 57ac4b25..8c657eec 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.10.3a1" +version = "1.10.2" From 740a8d6aacaea207e6b5e525f0d5b8164efa338a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 22 Nov 2024 19:37:02 -0500 Subject: [PATCH 34/63] revert pull_request trigger that was used for testing --- .github/workflows/pull-request-checks.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index e6c7de36..354995d5 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,8 +1,6 @@ name: "Pull request checks" on: - pull_request: - types: [opened, reopened, synchronize] pull_request_target: types: [opened, reopened, synchronize] From 32eab3a4cbd385c9b2297d650d1be884448eccff Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 4 Dec 2024 20:31:39 -0500 Subject: [PATCH 35/63] add in changelog entry check --- .github/workflows/_changelog-entry-check.yml | 82 ++++++++++++-------- .github/workflows/pull-request-checks.yml | 7 +- 2 files changed, 57 insertions(+), 32 deletions(-) diff --git a/.github/workflows/_changelog-entry-check.yml b/.github/workflows/_changelog-entry-check.yml index 8732177f..93f453c0 100644 --- a/.github/workflows/_changelog-entry-check.yml +++ b/.github/workflows/_changelog-entry-check.yml @@ -1,37 +1,57 @@ -# **what?** -# Checks that a file has been committed under the /.changes directory -# as a new CHANGELOG entry. Cannot check for a specific filename as -# it is dynamically generated by change type and timestamp. -# This workflow runs on pull_request_target because it requires -# secrets to post comments. - -# **why?** -# Ensure code change gets reflected in the CHANGELOG. - -# **when?** -# This will run for all PRs going into main. It will -# run when they are opened, reopened, when any label is added or removed -# and when new code is pushed to the branch. The action will get -# skipped if the 'Skip Changelog' label is present is any of the labels. - -name: Check Changelog Entry +name: "Changelog entry check" on: - pull_request_target: - types: [opened, reopened, labeled, unlabeled, synchronize] - -defaults: - run: - shell: bash + workflow_call: + inputs: + package: + description: "Choose the package to test" + type: string + default: "dbt-adapters" + issue: + description: "The issue/PR number" + type: string + required: true permissions: - contents: read - pull-requests: write + contents: read + pull-requests: write jobs: - changelog: - uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main - with: - changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' - skip_label: 'Skip Changelog' - secrets: inherit + changelog-check: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'Skip Changelog') }} + outputs: + exists: ${{ steps.changelog.outputs.exists }} + runs-on: ubuntu-latest + steps: + - id: changelog + uses: dorny/paths-filter@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + filters: | + exists: + - added|modified: '${{ inputs.package }}/.changes/unreleased/**.yaml' + + comment: + needs: changelog-check + if: needs.changelog-check.outputs.exists == false + runs-on: ubuntu-latest + env: + COMMENT: >- + Thank you for your pull request! We could not find a changelog entry for this change. + For details on how to document a change, see the + [Contributing Guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md). + steps: + - id: comment + uses: peter-evans/find-comment@v3 + with: + issue-number: ${{ inputs.issue }} + comment-author: "github-actions[bot]" + body-includes: ${{ env.COMMENT }} + - if: steps.comment.outputs.comment-body == '' + run: gh issue comment ${{ inputs.issue }} --repo ${{ github.repository }} --body "${{ env.COMMENT }}" + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - uses: actions/github-script@v7 + with: + script: core.setFailed('Changelog entry required to merge.') diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 354995d5..7f345a04 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -2,7 +2,7 @@ name: "Pull request checks" on: pull_request_target: - types: [opened, reopened, synchronize] + types: [opened, reopened, synchronize, labeled, unlabeled] # only run this once per PR at a time concurrency: @@ -10,6 +10,11 @@ concurrency: cancel-in-progress: true jobs: + changelog-entry: + uses: ./.github/workflows/_changelog-entry-check.yml + with: + issue: ${{ github.event.pull_request.number }} + code-quality: uses: ./.github/workflows/_code-quality.yml with: From 1142f3cfe631a8ed4830a733d3e5348dd900d3f4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 6 Dec 2024 18:22:19 -0500 Subject: [PATCH 36/63] remove duplicate action --- .github/workflows/changelog-exists.yml | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 .github/workflows/changelog-exists.yml diff --git a/.github/workflows/changelog-exists.yml b/.github/workflows/changelog-exists.yml deleted file mode 100644 index 1b5a2423..00000000 --- a/.github/workflows/changelog-exists.yml +++ /dev/null @@ -1,21 +0,0 @@ -name: "Check changelog entry exists" - -on: - pull_request_target: - types: [opened, reopened, labeled, unlabeled, synchronize] - -defaults: - run: - shell: bash - -permissions: - contents: read - pull-requests: write - -jobs: - changelog: - uses: dbt-labs/actions/.github/workflows/changelog-existence.yml@main - with: - changelog_comment: 'Thank you for your pull request! We could not find a changelog entry for this change. For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md#adding-changelog-entry).' - skip_label: 'Skip Changelog' - secrets: inherit From 6eb3a2634560ddae37c8613835a4447ccbd1cb60 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 6 Dec 2024 18:39:46 -0500 Subject: [PATCH 37/63] update changelog check to use the correct subdirectory --- .github/workflows/_changelog-entry-check.yml | 24 ++++++++++++-------- .github/workflows/_package-directory.yml | 7 +++++- .github/workflows/pull-request-checks.yml | 5 ++-- 3 files changed, 24 insertions(+), 12 deletions(-) diff --git a/.github/workflows/_changelog-entry-check.yml b/.github/workflows/_changelog-entry-check.yml index 93f453c0..22791043 100644 --- a/.github/workflows/_changelog-entry-check.yml +++ b/.github/workflows/_changelog-entry-check.yml @@ -7,8 +7,8 @@ on: description: "Choose the package to test" type: string default: "dbt-adapters" - issue: - description: "The issue/PR number" + pull-request: + description: "The PR number" type: string required: true @@ -17,7 +17,13 @@ permissions: pull-requests: write jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + changelog-check: + needs: package if: ${{ !contains(github.event.pull_request.labels.*.name, 'Skip Changelog') }} outputs: exists: ${{ steps.changelog.outputs.exists }} @@ -29,26 +35,26 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} filters: | exists: - - added|modified: '${{ inputs.package }}/.changes/unreleased/**.yaml' + - added|modified: '${{ needs.package.outputs.directory }}/.changes/unreleased/**.yaml' comment: needs: changelog-check if: needs.changelog-check.outputs.exists == false runs-on: ubuntu-latest env: + AUTHOR: "github-actions[bot]" COMMENT: >- - Thank you for your pull request! We could not find a changelog entry for this change. - For details on how to document a change, see the - [Contributing Guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md). + Thank you for your pull request! We could not find a changelog entry for this change in the ${{ inputs.package }} package. + For details on how to document a change, see the [Contributing Guide](https://github.com/dbt-labs/dbt-adapters/blob/main/CONTRIBUTING.md). steps: - id: comment uses: peter-evans/find-comment@v3 with: - issue-number: ${{ inputs.issue }} - comment-author: "github-actions[bot]" + issue-number: ${{ inputs.pull-request }} + comment-author: ${{ env.AUTHOR }} body-includes: ${{ env.COMMENT }} - if: steps.comment.outputs.comment-body == '' - run: gh issue comment ${{ inputs.issue }} --repo ${{ github.repository }} --body "${{ env.COMMENT }}" + run: gh issue comment ${{ inputs.pull-request }} --repo ${{ github.repository }} --body "${{ env.COMMENT }}" shell: bash env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 115abdfe..194b8f45 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -7,6 +7,11 @@ on: description: "Choose the package whose directory you need" type: string default: "dbt-adapters" + outputs: + directory: + description: "The root directory of the package" + type: string + value: ${{ jobs.package.outputs.directory }} jobs: package: @@ -19,7 +24,7 @@ jobs: run: | if [[ package == 'dbt-adapters' ]] then - directory="" + directory="." else directory="${{ inputs.package }}" fi diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 7f345a04..8ec3d0e6 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -13,7 +13,7 @@ jobs: changelog-entry: uses: ./.github/workflows/_changelog-entry-check.yml with: - issue: ${{ github.event.pull_request.number }} + pull-request: ${{ github.event.pull_request.number }} code-quality: uses: ./.github/workflows/_code-quality.yml @@ -48,9 +48,10 @@ jobs: results: name: "Pull request checks" # keep this name, branch protection references it if: always() - needs: [code-quality, verify-builds, unit-tests] + needs: [changelog-entry, code-quality, verify-builds, unit-tests] runs-on: ${{ vars.DEFAULT_RUNNER }} steps: - uses: re-actors/alls-green@release/v1 with: jobs: ${{ toJSON(needs) }} + allowed-skips: 'changelog-entry' From e0fe7f7cbe570c641efdf136ea02e8bc7c6df6b8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 6 Dec 2024 18:40:50 -0500 Subject: [PATCH 38/63] add dispatch trigger to changelog check --- .github/workflows/_changelog-entry-check.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/_changelog-entry-check.yml b/.github/workflows/_changelog-entry-check.yml index 22791043..ff31de6d 100644 --- a/.github/workflows/_changelog-entry-check.yml +++ b/.github/workflows/_changelog-entry-check.yml @@ -11,6 +11,16 @@ on: description: "The PR number" type: string required: true + workflow_dispatch: + inputs: + package: + description: "Choose the package to test" + type: string + default: "dbt-adapters" + pull-request: + description: "The PR number" + type: string + required: true permissions: contents: read From 77083a27de0f272ed06caed6c5d5a29cd2653df0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 6 Dec 2024 18:44:18 -0500 Subject: [PATCH 39/63] update runner to reference the default runner env var --- .github/workflows/_changelog-entry-check.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_changelog-entry-check.yml b/.github/workflows/_changelog-entry-check.yml index ff31de6d..a8cb882c 100644 --- a/.github/workflows/_changelog-entry-check.yml +++ b/.github/workflows/_changelog-entry-check.yml @@ -37,7 +37,7 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'Skip Changelog') }} outputs: exists: ${{ steps.changelog.outputs.exists }} - runs-on: ubuntu-latest + runs-on: ${{ vars.DEFAULT_RUNNER }} steps: - id: changelog uses: dorny/paths-filter@v3 @@ -50,7 +50,7 @@ jobs: comment: needs: changelog-check if: needs.changelog-check.outputs.exists == false - runs-on: ubuntu-latest + runs-on: ${{ vars.DEFAULT_RUNNER }} env: AUTHOR: "github-actions[bot]" COMMENT: >- From b0dde3fbc770b9c3c4010c47042ad8cb8bfedd05 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Fri, 6 Dec 2024 18:44:45 -0500 Subject: [PATCH 40/63] remove rebasing artifact --- .github/workflows/_code-quality.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/_code-quality.yml b/.github/workflows/_code-quality.yml index d3ddfc05..05d2e6da 100644 --- a/.github/workflows/_code-quality.yml +++ b/.github/workflows/_code-quality.yml @@ -1,7 +1,6 @@ name: "Code quality" on: - workflow_dispatch: workflow_call: inputs: branch: From 96325a23d6c46a55011287fc0268c562171e8e08 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 02:04:53 -0500 Subject: [PATCH 41/63] generalize changelog inputs --- .github/workflows/_generate-changelog.yml | 24 +++++++++++------------ .github/workflows/publish.yml | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml index aca712b2..37b67bbb 100644 --- a/.github/workflows/_generate-changelog.yml +++ b/.github/workflows/_generate-changelog.yml @@ -7,10 +7,10 @@ on: description: "Choose the package getting published" type: string default: "dbt-adapters" - deploy-to: - description: "Choose whether we're publishing to test or prod" - type: string - default: "prod" + merge: + description: "Choose whether to merge the changelog branch" + type: boolean + default: true branch: description: "Choose the branch to use" type: string @@ -32,10 +32,10 @@ on: description: "Choose the package getting published" type: string default: "dbt-adapters" - deploy-to: - description: "Choose whether we're publishing to test or prod" - type: environment - default: "prod" + merge: + description: "Choose whether to merge the changelog branch" + type: boolean + default: false branch: description: "Choose the branch to use" type: string @@ -86,8 +86,8 @@ jobs: version: ${{ steps.version.outputs.raw }} changelog: - runs-on: ${{ vars.DEFAULT_RUNNER }} needs: [package, version] + runs-on: ${{ vars.DEFAULT_RUNNER }} outputs: path: ${{ steps.changelog.outputs.path }} exists: ${{ steps.changelog.outputs.exists }} @@ -194,9 +194,9 @@ jobs: working-directory: ./${{ needs.package.outputs.directory }} merge-changes: - runs-on: ${{ vars.DEFAULT_RUNNER }} needs: [temp-branch, generate-changelog] - if: ${{ needs.temp-branch.outputs.name != '' && inputs.deploy-to == 'prod' }} + if: ${{ needs.temp-branch.outputs.name != '' && inputs.merge }} + runs-on: ${{ vars.DEFAULT_RUNNER }} steps: - uses: everlytic/branch-merge@1.1.5 with: @@ -220,7 +220,7 @@ jobs: - id: branch run: | branch="" - if [ ${{ needs.temp-branch.outputs.name == '' || inputs.deploy-to == 'prod' }}] + if [ ${{ needs.temp-branch.outputs.name == '' || inputs.merge }}] then branch="${{ inputs.branch }}" else diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5fe98574..14f8653c 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -43,7 +43,7 @@ jobs: uses: ./.github/workflows/_generate-changelog.yml with: package: ${{ inputs.package }} - deploy-to: ${{ inputs.deploy-to }} + merge: ${{ inputs.deploy-to == 'prod' }} branch: ${{ inputs.branch }} secrets: inherit From d5981da23a0ea392919c650674674077b05e0c19 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 02:22:47 -0500 Subject: [PATCH 42/63] account for adapter subpackage --- .github/workflows/_unit-tests.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index 7e6b40f8..0c0a8215 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -50,7 +50,13 @@ permissions: contents: read jobs: + package: + uses: ./.github/workflows/_package-directory.yml + with: + package: ${{ inputs.package }} + unit-tests: + needs: package runs-on: ${{ inputs.os }} steps: - uses: actions/checkout@v4 @@ -63,3 +69,4 @@ jobs: - uses: pypa/hatch@install - run: hatch run unit-tests shell: bash + working-directory: ./${{ needs.package.outputs.directory }} From 67c803d1580dd1aa19e8832deefe8315daef4b54 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 02:23:19 -0500 Subject: [PATCH 43/63] allow for running against a fork --- .github/workflows/_verify-build.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml index a5c33030..09a105d1 100644 --- a/.github/workflows/_verify-build.yml +++ b/.github/workflows/_verify-build.yml @@ -48,10 +48,6 @@ on: permissions: read-all -defaults: - run: - shell: bash - jobs: package: uses: ./.github/workflows/_package-directory.yml @@ -65,9 +61,11 @@ jobs: - uses: actions/checkout@v4 with: ref: ${{ inputs.branch }} + repository: ${{ inputs.repository }} - uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} - uses: pypa/hatch@install - run: hatch build && hatch run build:check-all + shell: bash working-directory: ./${{ needs.package.outputs.directory }} From 0bfa6525b5eeb2e940d1e9a959c0fe162a4ec5dc Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 11:28:05 -0500 Subject: [PATCH 44/63] add run names and concurrency conditions to all workflows --- .github/workflows/issue-triage.yml | 1 + .github/workflows/publish.yml | 3 ++- .github/workflows/pull-request-checks.yml | 1 + .github/workflows/scheduled-maintenance.yml | 4 ++-- .github/workflows/user-docs.yml | 18 +++++++++--------- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.github/workflows/issue-triage.yml b/.github/workflows/issue-triage.yml index 5432121f..7903a732 100644 --- a/.github/workflows/issue-triage.yml +++ b/.github/workflows/issue-triage.yml @@ -1,4 +1,5 @@ name: "Issue triage" +run-name: "Issue triage - #${{ github.event.issue.number }}: ${{ github.event.issue.title }} - ${{ github.actor }}" on: issue_comment diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 14f8653c..04a14545 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -1,4 +1,5 @@ name: "Publish" +run-name: "Publish - ${{ inputs.package }} - ${{ inputs.deploy-to }} - ${{ github.actor }}" on: workflow_dispatch: @@ -26,7 +27,7 @@ on: type: boolean default: false -# don't attempt to release the same target in parallel +# don't publish to the same target in parallel concurrency: group: ${{ github.workflow }}-${{ inputs.package }}-${{ inputs.deploy-to }} cancel-in-progress: true diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 8ec3d0e6..0fd958ee 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -1,4 +1,5 @@ name: "Pull request checks" +run-name: "Publish - #${{ github.event.number }} - ${{ github.actor }}" on: pull_request_target: diff --git a/.github/workflows/scheduled-maintenance.yml b/.github/workflows/scheduled-maintenance.yml index 28309b95..6e18e668 100644 --- a/.github/workflows/scheduled-maintenance.yml +++ b/.github/workflows/scheduled-maintenance.yml @@ -3,13 +3,13 @@ name: "Scheduled maintenance" on: schedule: - cron: "30 1 * * SAT" - workflow_dispatch: permissions: contents: write +# don't run this in parallel concurrency: - group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.sha }} + group: ${{ github.workflow }} cancel-in-progress: true jobs: diff --git a/.github/workflows/user-docs.yml b/.github/workflows/user-docs.yml index 66bd3da1..065d1d2e 100644 --- a/.github/workflows/user-docs.yml +++ b/.github/workflows/user-docs.yml @@ -1,4 +1,5 @@ name: "Open user docs issue" +run-name: "Open user docs issue - #${{ github.event.issue.number }} - ${{ github.actor }}" on: issues: @@ -11,18 +12,17 @@ defaults: permissions: issues: write # comments on issues +# only run this once per issue +concurrency: + group: ${{ github.workflow }}-${{ github.event.issue.number }} + cancel-in-progress: true + jobs: open_issues: - # we only want to run this when the issue is closed as completed and the label `user docs` has been assigned. - # If this logic does not exist in this workflow, it runs the - # risk of duplicaton of issues being created due to merge and label both triggering this workflow to run and neither having - # generating the comment before the other runs. This lives here instead of the shared workflow because this is where we - # decide if it should run or not. if: | - (github.event.issue.state == 'closed' && github.event.issue.state_reason == 'completed') && ( - (github.event.action == 'closed' && contains(github.event.issue.labels.*.name, 'user docs')) || - (github.event.action == 'labeled' && github.event.label.name == 'user docs') - ) + github.event.issue.state == 'closed' && + github.event.issue.state_reason == 'completed' && + contains(github.event.issue.labels.*.name, 'user docs') uses: dbt-labs/actions/.github/workflows/open-issue-in-repo.yml@main with: issue_repository: "dbt-labs/docs.getdbt.com" From 39b6ffc108e77e735823c513b8cbc5ba0035ba79 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 11:47:10 -0500 Subject: [PATCH 45/63] pull stale workflow local to update labels --- .github/workflows/scheduled-maintenance.yml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/scheduled-maintenance.yml b/.github/workflows/scheduled-maintenance.yml index 6e18e668..d0b6fe29 100644 --- a/.github/workflows/scheduled-maintenance.yml +++ b/.github/workflows/scheduled-maintenance.yml @@ -21,4 +21,21 @@ jobs: SLACK_WEBHOOK_ALERTS_URL: ${{ secrets.SLACK_DEV_ADAPTER_ALERTS }} stale: - uses: dbt-labs/actions/.github/workflows/stale-bot-matrix.yml@main + runs-on: ${{ vars.DEFAULT_RUNNER }} + strategy: + matrix: + include: + - threshold: 90 + labels: 'triage:awaiting-response,triage:more-information-needed' + - threshold: 360 + labels: 'misc:good-first-issue,misc:help-wanted,type:tech-debt' + steps: + - uses: actions/stale@v9 + with: + stale-issue-message: "This issue has been marked as Stale because it has been open for 180 days with no activity. If you would like the issue to remain open, please comment on the issue or else it will be closed in 7 days." + stale-pr-message: "This PR has been marked as Stale because it has been open with no activity as of late. If you would like the PR to remain open, please comment on the PR or else it will be closed in 7 days." + close-issue-message: "Although we are closing this issue as stale, it's not gone forever. Issues can be reopened if there is renewed community interest. Just add a comment to notify the maintainers." + close-pr-message: "Although we are closing this PR as stale, it can still be reopened to continue development. Just add a comment to notify the maintainers." + close-issue-reason: "not_planned" + days-before-stale: ${{ matrix.threshold }} + any-of-labels: ${{ matrix.labels }} From 849c6b84c6a14cc91b97024546c5d1ac6556eea7 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 11:51:13 -0500 Subject: [PATCH 46/63] update format to match other workflows --- .github/workflows/_verify-build.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml index 09a105d1..f667b600 100644 --- a/.github/workflows/_verify-build.yml +++ b/.github/workflows/_verify-build.yml @@ -28,7 +28,9 @@ on: package: description: "Choose the package to build" type: choice - options: ["dbt-adapters", "dbt-tests-adapter"] + options: + - "dbt-adapters" + - "dbt-tests-adapter" branch: description: "Choose the branch to build" type: string From 0ecccb5be1a0a389c5da4ccabbaba88d473414b7 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:08:16 -0500 Subject: [PATCH 47/63] register atomic workflows by adding a pull_request trigger; to be removed after registering --- .github/workflows/_changelog-entry-check.yml | 1 + .github/workflows/_code-quality.yml | 1 + .github/workflows/_generate-changelog.yml | 1 + .github/workflows/_package-directory.yml | 1 + .github/workflows/_publish-internal.yml | 1 + .github/workflows/_publish-pypi.yml | 1 + .github/workflows/_unit-tests.yml | 1 + .github/workflows/_verify-build.yml | 1 + 8 files changed, 8 insertions(+) diff --git a/.github/workflows/_changelog-entry-check.yml b/.github/workflows/_changelog-entry-check.yml index a8cb882c..7bd07a7d 100644 --- a/.github/workflows/_changelog-entry-check.yml +++ b/.github/workflows/_changelog-entry-check.yml @@ -1,6 +1,7 @@ name: "Changelog entry check" on: + pull_request: workflow_call: inputs: package: diff --git a/.github/workflows/_code-quality.yml b/.github/workflows/_code-quality.yml index 05d2e6da..afe4baf2 100644 --- a/.github/workflows/_code-quality.yml +++ b/.github/workflows/_code-quality.yml @@ -1,6 +1,7 @@ name: "Code quality" on: + pull_request: workflow_call: inputs: branch: diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml index 37b67bbb..ea2d7a15 100644 --- a/.github/workflows/_generate-changelog.yml +++ b/.github/workflows/_generate-changelog.yml @@ -1,6 +1,7 @@ name: "Changelog generation" on: + pull_request: workflow_call: inputs: package: diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 194b8f45..e85dde3a 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -1,6 +1,7 @@ name: "Package directory" on: + pull_request: workflow_call: inputs: package: diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml index 1c339701..080c43f2 100644 --- a/.github/workflows/_publish-internal.yml +++ b/.github/workflows/_publish-internal.yml @@ -1,6 +1,7 @@ name: "Publish internally" on: + pull_request: workflow_call: inputs: package: diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml index 4150631d..325bb34a 100644 --- a/.github/workflows/_publish-pypi.yml +++ b/.github/workflows/_publish-pypi.yml @@ -1,6 +1,7 @@ name: "Publish to PyPI" on: + pull_request: workflow_call: inputs: package: diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index 0c0a8215..df8df1c8 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -1,6 +1,7 @@ name: "Unit tests" on: + pull_request: workflow_call: inputs: package: diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml index f667b600..0305cc38 100644 --- a/.github/workflows/_verify-build.yml +++ b/.github/workflows/_verify-build.yml @@ -1,6 +1,7 @@ name: "Verify build" on: + pull_request: workflow_call: inputs: package: From c468e541ae832afa791f0e340d4d7522d8d76109 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:11:14 -0500 Subject: [PATCH 48/63] register atomic workflows by adding a pull_request trigger; to be removed after registering --- .github/workflows/pull-request-checks.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 0fd958ee..f00aa394 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -2,6 +2,7 @@ name: "Pull request checks" run-name: "Publish - #${{ github.event.number }} - ${{ github.actor }}" on: + pull_request: pull_request_target: types: [opened, reopened, synchronize, labeled, unlabeled] From febbb9cb385a08685399689a9ed50e1fddeea361 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:12:35 -0500 Subject: [PATCH 49/63] fix version --- dbt/adapters/__about__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 8c657eec..08e0d06b 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.10.2" +version = "1.10.4" From 9f257a75e4b3440ddf3cce83598924b592a4f5be Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:15:29 -0500 Subject: [PATCH 50/63] fix package directory outputs --- .github/workflows/_package-directory.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index e85dde3a..4199c24f 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -11,7 +11,6 @@ on: outputs: directory: description: "The root directory of the package" - type: string value: ${{ jobs.package.outputs.directory }} jobs: From 59c68b8362622590f60edc20059c675be3311974 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:24:26 -0500 Subject: [PATCH 51/63] fix package directory outputs --- .github/workflows/_package-directory.yml | 2 +- pyproject.toml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 4199c24f..53961334 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -22,7 +22,7 @@ jobs: - id: package shell: bash run: | - if [[ package == 'dbt-adapters' ]] + if [ package == 'dbt-adapters' ] then directory="." else diff --git a/pyproject.toml b/pyproject.toml index 36c63556..0fef004a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,6 +64,8 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" + +test-changelog-entry-check = "gh workflow run _changelog-entry-check.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f pull-request={args}" test-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" [tool.hatch.envs.build] From ba65a2453427086490ce8698cc9fb6a469834426 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:29:06 -0500 Subject: [PATCH 52/63] fix package directory outputs --- .github/workflows/_package-directory.yml | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 53961334..81a73e76 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -13,6 +13,10 @@ on: description: "The root directory of the package" value: ${{ jobs.package.outputs.directory }} +defaults: + run: + shell: bash + jobs: package: runs-on: ${{ vars.DEFAULT_RUNNER }} @@ -20,12 +24,15 @@ jobs: directory: ${{ steps.package.outputs.directory }} steps: - id: package - shell: bash run: | - if [ package == 'dbt-adapters' ] + if [[ package == "dbt-adapters" ]] then directory="." else directory="${{ inputs.package }}" fi echo "directory=$directory" >> $GITHUB_OUTPUT + - run: echo "::notice title=$TITLE::$MESSAGE" + env: + TITLE: "Package directory" + MESSAGE: "${{ steps.pacakge.outputs.directory }}" From 82176517023011bb945a782d30163d12064e48de Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:30:57 -0500 Subject: [PATCH 53/63] fix package directory input --- .github/workflows/_package-directory.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 81a73e76..de7df975 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -25,7 +25,7 @@ jobs: steps: - id: package run: | - if [[ package == "dbt-adapters" ]] + if [[ ${{ inputs.package }} == "dbt-adapters" ]] then directory="." else From 3dfa15829d7ef6e5520aafc16e278c39df4a4c8c Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:37:51 -0500 Subject: [PATCH 54/63] fix package directory --- .github/workflows/_changelog-entry-check.yml | 2 +- .github/workflows/_package-directory.yml | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_changelog-entry-check.yml b/.github/workflows/_changelog-entry-check.yml index 7bd07a7d..3e2c324f 100644 --- a/.github/workflows/_changelog-entry-check.yml +++ b/.github/workflows/_changelog-entry-check.yml @@ -46,7 +46,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} filters: | exists: - - added|modified: '${{ needs.package.outputs.directory }}/.changes/unreleased/**.yaml' + - added|modified: '${{ needs.package.outputs.directory }}.changes/unreleased/**.yaml' comment: needs: changelog-check diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index de7df975..92f97f88 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -27,12 +27,13 @@ jobs: run: | if [[ ${{ inputs.package }} == "dbt-adapters" ]] then - directory="." + directory="" else - directory="${{ inputs.package }}" + directory="${{ inputs.package }}/" fi echo "directory=$directory" >> $GITHUB_OUTPUT - - run: echo "::notice title=$TITLE::$MESSAGE" + - run: | + echo "::notice $TITLE::$MESSAGE" env: TITLE: "Package directory" MESSAGE: "${{ steps.pacakge.outputs.directory }}" From a3bf5b85b0ac856a94daf4abfea3d88181e4caf8 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 12:53:11 -0500 Subject: [PATCH 55/63] remove workflow dispatch testing for changelog-entry-check since it is not possible --- .github/workflows/_changelog-entry-check.yml | 13 ++----------- pyproject.toml | 2 -- 2 files changed, 2 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_changelog-entry-check.yml b/.github/workflows/_changelog-entry-check.yml index 3e2c324f..ddc97290 100644 --- a/.github/workflows/_changelog-entry-check.yml +++ b/.github/workflows/_changelog-entry-check.yml @@ -1,7 +1,8 @@ name: "Changelog entry check" +# this cannot be tested via workflow_dispatch +# dorny/paths-filter inspects the current trigger to determine how to compare branches on: - pull_request: workflow_call: inputs: package: @@ -12,16 +13,6 @@ on: description: "The PR number" type: string required: true - workflow_dispatch: - inputs: - package: - description: "Choose the package to test" - type: string - default: "dbt-adapters" - pull-request: - description: "The PR number" - type: string - required: true permissions: contents: read diff --git a/pyproject.toml b/pyproject.toml index 0fef004a..36c63556 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,8 +64,6 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" - -test-changelog-entry-check = "gh workflow run _changelog-entry-check.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f pull-request={args}" test-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" [tool.hatch.envs.build] From 68c8b5797f3464ac3f97846d97ecc60680944211 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 13:00:34 -0500 Subject: [PATCH 56/63] add workflow test script for code quality --- .github/workflows/_code-quality.yml | 1 - pyproject.toml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_code-quality.yml b/.github/workflows/_code-quality.yml index afe4baf2..05d2e6da 100644 --- a/.github/workflows/_code-quality.yml +++ b/.github/workflows/_code-quality.yml @@ -1,7 +1,6 @@ name: "Code quality" on: - pull_request: workflow_call: inputs: branch: diff --git a/pyproject.toml b/pyproject.toml index 36c63556..1f09cf4b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,8 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" -test-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" +workflow-code-quality = "gh workflow run _code-quality.yml --ref $(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD)" +workflow-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" [tool.hatch.envs.build] detached = true From c4249c5a1b69853a43b9f29587c9bb9ee051dc6a Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 13:06:06 -0500 Subject: [PATCH 57/63] add workflow test script for unit tests --- .github/workflows/_unit-tests.yml | 1 - pyproject.toml | 3 ++- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/_unit-tests.yml b/.github/workflows/_unit-tests.yml index df8df1c8..0c0a8215 100644 --- a/.github/workflows/_unit-tests.yml +++ b/.github/workflows/_unit-tests.yml @@ -1,7 +1,6 @@ name: "Unit tests" on: - pull_request: workflow_call: inputs: package: diff --git a/pyproject.toml b/pyproject.toml index 1f09cf4b..b2070f3e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,7 +64,8 @@ dependencies = [ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" -workflow-code-quality = "gh workflow run _code-quality.yml --ref $(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD)" +workflow-code-quality = "gh workflow run _code-quality.yml --ref $(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD)" +workflow-unit-tests = "gh workflow run _unit-tests.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" [tool.hatch.envs.build] From 6a5062d299cbda7aa4809f97ec4b27610b36efe4 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 13:10:37 -0500 Subject: [PATCH 58/63] add workflow test script for verify build --- .github/workflows/_verify-build.yml | 1 - pyproject.toml | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_verify-build.yml b/.github/workflows/_verify-build.yml index 0305cc38..f667b600 100644 --- a/.github/workflows/_verify-build.yml +++ b/.github/workflows/_verify-build.yml @@ -1,7 +1,6 @@ name: "Verify build" on: - pull_request: workflow_call: inputs: package: diff --git a/pyproject.toml b/pyproject.toml index b2070f3e..3344cb02 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,7 @@ code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" workflow-code-quality = "gh workflow run _code-quality.yml --ref $(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-unit-tests = "gh workflow run _unit-tests.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" +workflow-verify-build = "gh workflow run _verify-build.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" [tool.hatch.envs.build] From e5e2fedcfe84b5c1b0e2ecdf691ae1982d6d7515 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 13:17:31 -0500 Subject: [PATCH 59/63] bump version to test changelog generation --- .github/workflows/_package-directory.yml | 5 ----- dbt/adapters/__about__.py | 2 +- pyproject.toml | 1 + 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 92f97f88..9663d399 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -32,8 +32,3 @@ jobs: directory="${{ inputs.package }}/" fi echo "directory=$directory" >> $GITHUB_OUTPUT - - run: | - echo "::notice $TITLE::$MESSAGE" - env: - TITLE: "Package directory" - MESSAGE: "${{ steps.pacakge.outputs.directory }}" diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 08e0d06b..5ec8c2e3 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.10.4" +version = "1.10.5" diff --git a/pyproject.toml b/pyproject.toml index 3344cb02..81b6420e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -65,6 +65,7 @@ setup = "pre-commit install" code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" workflow-code-quality = "gh workflow run _code-quality.yml --ref $(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD)" +workflow-generate-changelog = "gh workflow run _generate-changelog.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f merge=false -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-unit-tests = "gh workflow run _unit-tests.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-verify-build = "gh workflow run _verify-build.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" From 8782e477006983cb7facc41ebe3c128a8fd56893 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 13:21:37 -0500 Subject: [PATCH 60/63] bump version to test changelog generation --- .github/workflows/_generate-changelog.yml | 1 - dbt/adapters/__about__.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/_generate-changelog.yml b/.github/workflows/_generate-changelog.yml index ea2d7a15..37b67bbb 100644 --- a/.github/workflows/_generate-changelog.yml +++ b/.github/workflows/_generate-changelog.yml @@ -1,7 +1,6 @@ name: "Changelog generation" on: - pull_request: workflow_call: inputs: package: diff --git a/dbt/adapters/__about__.py b/dbt/adapters/__about__.py index 5ec8c2e3..08e0d06b 100644 --- a/dbt/adapters/__about__.py +++ b/dbt/adapters/__about__.py @@ -1 +1 @@ -version = "1.10.5" +version = "1.10.4" From 1d3676609cd6098e206a94216889da7630137d26 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 13:46:41 -0500 Subject: [PATCH 61/63] fix version things for publishing --- .github/workflows/_package-directory.yml | 8 +++++++- .github/workflows/_publish-internal.yml | 3 +-- .github/workflows/_publish-pypi.yml | 3 +-- pyproject.toml | 2 ++ 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index 9663d399..bbfc9123 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -1,7 +1,6 @@ name: "Package directory" on: - pull_request: workflow_call: inputs: package: @@ -12,6 +11,9 @@ on: directory: description: "The root directory of the package" value: ${{ jobs.package.outputs.directory }} + version-file: + description: "The path to the version file of the package" + value: ${{ jobs.package.outputs.version-file }} defaults: run: @@ -22,13 +24,17 @@ jobs: runs-on: ${{ vars.DEFAULT_RUNNER }} outputs: directory: ${{ steps.package.outputs.directory }} + version-file: ${{ steps.package.outputs.version-file }} steps: - id: package run: | if [[ ${{ inputs.package }} == "dbt-adapters" ]] then directory="" + version_file="./dbt/adapters/__about__.py" else directory="${{ inputs.package }}/" + version_file="./src/dbt/adapters/$(cut -c 5- ${{ inputs.package }})/__version__.py" fi echo "directory=$directory" >> $GITHUB_OUTPUT + echo "version-file=version_file" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml index 080c43f2..a24971ff 100644 --- a/.github/workflows/_publish-internal.yml +++ b/.github/workflows/_publish-internal.yml @@ -1,7 +1,6 @@ name: "Publish internally" on: - pull_request: workflow_call: inputs: package: @@ -82,7 +81,7 @@ jobs: versions_published: ${{ steps.published.outputs.versions }} - run: | VERSION=${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD) - tee <<< "version = \"$VERSION\"" ./src/dbt/adapters/athena/__version__.py + tee <<< "version = \"$VERSION\"" ${{ needs.package.outputs.version-file }} working-directory: ./${{ needs.package.outputs.directory }} - run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml working-directory: ./${{ needs.package.outputs.directory }} diff --git a/.github/workflows/_publish-pypi.yml b/.github/workflows/_publish-pypi.yml index 325bb34a..f05b1fd6 100644 --- a/.github/workflows/_publish-pypi.yml +++ b/.github/workflows/_publish-pypi.yml @@ -1,7 +1,6 @@ name: "Publish to PyPI" on: - pull_request: workflow_call: inputs: package: @@ -69,7 +68,7 @@ jobs: - uses: pypa/gh-action-pypi-publish@release/v1 with: repository-url: ${{ vars.PYPI_REPOSITORY_URL }} - packages-dir: ${{ inputs.package }}/dist/ + packages-dir: ./${{ needs.package.outputs.directory }}dist/ verify: runs-on: ${{ vars.DEFAULT_RUNNER }} diff --git a/pyproject.toml b/pyproject.toml index 81b6420e..d84f35f8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,6 +66,8 @@ code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" workflow-code-quality = "gh workflow run _code-quality.yml --ref $(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-generate-changelog = "gh workflow run _generate-changelog.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f merge=false -f branch=$(git rev-parse --abbrev-ref HEAD)" +workflow-publish-internal = "gh workflow run _publish-internal.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f deploy-to=test -f branch=$(git rev-parse --abbrev-ref HEAD)" +workflow-publish-pypi = "gh workflow run _publish-pypi.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f deploy-to=test -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-unit-tests = "gh workflow run _unit-tests.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-verify-build = "gh workflow run _verify-build.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-publish = "gh workflow run publish.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD) -f deploy-to=test -f pypi-internal=false -f pypi-public=true" From 325f53496f22ca8e972c29a6cd36aabd9feaa1bb Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 13:58:22 -0500 Subject: [PATCH 62/63] remove internal release workaround for dbt-adapters since it is not currently published internally --- .github/workflows/_package-directory.yml | 7 ------- .github/workflows/_publish-internal.yml | 2 +- pyproject.toml | 1 - 3 files changed, 1 insertion(+), 9 deletions(-) diff --git a/.github/workflows/_package-directory.yml b/.github/workflows/_package-directory.yml index bbfc9123..d10e9758 100644 --- a/.github/workflows/_package-directory.yml +++ b/.github/workflows/_package-directory.yml @@ -11,9 +11,6 @@ on: directory: description: "The root directory of the package" value: ${{ jobs.package.outputs.directory }} - version-file: - description: "The path to the version file of the package" - value: ${{ jobs.package.outputs.version-file }} defaults: run: @@ -24,17 +21,13 @@ jobs: runs-on: ${{ vars.DEFAULT_RUNNER }} outputs: directory: ${{ steps.package.outputs.directory }} - version-file: ${{ steps.package.outputs.version-file }} steps: - id: package run: | if [[ ${{ inputs.package }} == "dbt-adapters" ]] then directory="" - version_file="./dbt/adapters/__about__.py" else directory="${{ inputs.package }}/" - version_file="./src/dbt/adapters/$(cut -c 5- ${{ inputs.package }})/__version__.py" fi echo "directory=$directory" >> $GITHUB_OUTPUT - echo "version-file=version_file" >> $GITHUB_OUTPUT diff --git a/.github/workflows/_publish-internal.yml b/.github/workflows/_publish-internal.yml index a24971ff..42fb6290 100644 --- a/.github/workflows/_publish-internal.yml +++ b/.github/workflows/_publish-internal.yml @@ -81,7 +81,7 @@ jobs: versions_published: ${{ steps.published.outputs.versions }} - run: | VERSION=${{ steps.next.outputs.internal_release_version }}+$(git rev-parse HEAD) - tee <<< "version = \"$VERSION\"" ${{ needs.package.outputs.version-file }} + tee <<< "version = \"$VERSION\"" ./src/dbt/adapters/$(cut -c 5- ${{ inputs.package }})/__version__.py working-directory: ./${{ needs.package.outputs.directory }} - run: sed -i "/dbt-core[<>~=]/d" ./pyproject.toml working-directory: ./${{ needs.package.outputs.directory }} diff --git a/pyproject.toml b/pyproject.toml index d84f35f8..0c03b341 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,6 @@ code-quality = "pre-commit run --all-files" unit-tests = "python -m pytest {args:tests/unit}" workflow-code-quality = "gh workflow run _code-quality.yml --ref $(git rev-parse --abbrev-ref HEAD) -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-generate-changelog = "gh workflow run _generate-changelog.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f merge=false -f branch=$(git rev-parse --abbrev-ref HEAD)" -workflow-publish-internal = "gh workflow run _publish-internal.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f deploy-to=test -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-publish-pypi = "gh workflow run _publish-pypi.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f deploy-to=test -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-unit-tests = "gh workflow run _unit-tests.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" workflow-verify-build = "gh workflow run _verify-build.yml --ref $(git rev-parse --abbrev-ref HEAD) -f package=dbt-adapters -f branch=$(git rev-parse --abbrev-ref HEAD)" From 33abd533afadadcbee54de226970a02407d94bb0 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Sat, 7 Dec 2024 14:10:58 -0500 Subject: [PATCH 63/63] remove pull request trigger on pull request checks, this should only run in the context of the target branch --- .github/workflows/pull-request-checks.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index f00aa394..0fd958ee 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -2,7 +2,6 @@ name: "Pull request checks" run-name: "Publish - #${{ github.event.number }} - ${{ github.actor }}" on: - pull_request: pull_request_target: types: [opened, reopened, synchronize, labeled, unlabeled]