Merge branch 'master' into tope/migrate-adjust-lowcode

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.58.0
+current_version = 0.58.1
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/CODEOWNERS

@@ -51,3 +51,10 @@
 /airbyte-integrations/connectors/destination-s3/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-snowflake/ @airbytehq/destinations
 /airbyte-integrations/connectors/destination-redshift/ @airbytehq/destinations
+
+# Python critical connectors
+/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/critical-connectors
+/airbyte-integrations/connectors/source-hubspot/ @airbytehq/critical-connectors
+/airbyte-integrations/connectors/source-salesforce/ @airbytehq/critical-connectors
+/airbyte-integrations/connectors/source-shopify/ @airbytehq/critical-connectors
+/airbyte-integrations/connectors/source-stripe/ @airbytehq/critical-connectors

.github/actions/install-airbyte-ci/action.yml

@@ -36,8 +36,10 @@ runs:
         if [[ "${{ github.ref }}" != "refs/heads/master" ]] && [[ "${{ steps.changes.outputs.pipelines_any_changed }}" == "true" ]]; then
           echo "Making changes to Airbyte CI on a non-master branch. Airbyte-CI will be installed from source."
           echo "install-mode=source" >> $GITHUB_OUTPUT
+          echo "SENTRY_ENVIRONMENT=dev" >> $GITHUB_ENV
         else
           echo "install-mode=binary" >> $GITHUB_OUTPUT
+          echo "SENTRY_ENVIRONMENT=production" >> $GITHUB_ENV
         fi
 
     - name: Install Airbyte CI from binary

.github/actions/run-airbyte-ci/action.yml

@@ -9,16 +9,16 @@ inputs:
     required: true
   github_token:
     description: "GitHub token"
-    required: true
+    required: false
   dagger_cloud_token:
     description: "Dagger Cloud token"
-    required: true
+    required: false
   docker_hub_username:
     description: "Dockerhub username"
-    required: true
+    required: false
   docker_hub_password:
     description: "Dockerhub password"
-    required: true
+    required: false
   options:
     description: "Options for the subcommand"
     required: false
@@ -93,15 +93,14 @@ runs:
     - name: Get start timestamp
       id: get-start-timestamp
       shell: bash
-      run: echo "name=start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
-
+      run: echo "start-timestamp=$(date +%s)" >> $GITHUB_OUTPUT
     - name: Docker login
       id: docker-login
       uses: docker/login-action@v3
+      if: ${{ inputs.docker_hub_username != '' && inputs.docker_hub_password != '' }}
       with:
         username: ${{ inputs.docker_hub_username }}
         password: ${{ inputs.docker_hub_password }}
-
     - name: Install Airbyte CI
       id: install-airbyte-ci
       uses: ./.github/actions/install-airbyte-ci
@@ -111,67 +110,68 @@ runs:
     - name: Run airbyte-ci
       id: run-airbyte-ci
       shell: bash
-      run: |
-        airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
       env:
+        CI: "True"
+        CI_GIT_USER: ${{ github.repository_owner }}
+        CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
+        PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
+        # Next environment variables are workflow inputs based and can be set with empty values if the inputs are not required and passed
         CI_CONTEXT: "${{ inputs.context }}"
-        CI_GIT_REPO_URL: ${{ inputs.git_repo_url }}
         CI_GIT_BRANCH: ${{ inputs.git_branch || github.head_ref }}
+        CI_GIT_REPO_URL: ${{ inputs.git_repo_url }}
         CI_GIT_REVISION: ${{ inputs.git_revision || github.sha }}
         CI_GITHUB_ACCESS_TOKEN: ${{ inputs.github_token }}
         CI_JOB_KEY: ${{ inputs.ci_job_key }}
-        CI_PIPELINE_START_TIMESTAMP: ${{ steps.get-start-timestamp.outputs.start-timestamp }}
         CI_REPORT_BUCKET_NAME: ${{ inputs.report_bucket_name }}
-        CI: "True"
         DAGGER_CLOUD_TOKEN: "${{ inputs.dagger_cloud_token }}"
         DOCKER_HUB_PASSWORD: ${{ inputs.docker_hub_password }}
         DOCKER_HUB_USERNAME: ${{ inputs.docker_hub_username }}
         GCP_GSM_CREDENTIALS: ${{ inputs.gcp_gsm_credentials }}
+        GCP_INTEGRATION_TESTER_CREDENTIALS: ${{ inputs.gcp_integration_tester_credentials }}
         GCS_CREDENTIALS: ${{ inputs.gcs_credentials }}
         METADATA_SERVICE_BUCKET_NAME: ${{ inputs.metadata_service_bucket_name }}
         METADATA_SERVICE_GCS_CREDENTIALS: ${{ inputs.metadata_service_gcs_credentials }}
         PRODUCTION: ${{ inputs.production }}
-        PULL_REQUEST_NUMBER: ${{ github.event.pull_request.number }}
         PYTHON_REGISTRY_TOKEN: ${{ inputs.python_registry_token }}
         PYTHON_REGISTRY_URL: ${{ inputs.python_registry_url }}
-        PYTHON_REGISTRY_CHECK_URL: ${{ inputs.python_registry_check_url }}
         S3_BUILD_CACHE_ACCESS_KEY_ID: ${{ inputs.s3_build_cache_access_key_id }}
         S3_BUILD_CACHE_SECRET_KEY: ${{ inputs.s3_build_cache_secret_key }}
         SENTRY_DSN: ${{ inputs.sentry_dsn }}
-        SENTRY_ENVIRONMENT: ${{ steps.determine-install-mode.outputs.install-mode }}
         SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
         SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
         SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}
-    # give the Dagger Engine more time to push cache data to Dagger Cloud
+      run: |
+        airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
     - name: Stop Engine
       id: stop-engine
       if: always()
       shell: bash
       run: |
         mapfile -t containers < <(docker ps --filter name="dagger-engine-*" -q)
         if [[ "${#containers[@]}" -gt 0 ]]; then
+          # give 5mn to the Dagger Engine to push cache data to Dagger Cloud
           docker stop -t 300 "${containers[@]}";
         fi
 
-    - name: Collect docker logs on failure
-      id: collect-docker-logs
+    - name: Collect dagger engine logs
+      id: collect-dagger-engine-logs
       if: always()
       uses: jwalton/gh-docker-logs@v2
       with:
-        dest: "./docker_logs"
+        dest: "./dagger_engine_logs"
         images: "registry.dagger.io/engine"
 
     - name: Tar logs
       id: tar-logs
       if: always()
       shell: bash
-      run: tar cvzf ./docker_logs.tgz ./docker_logs
+      run: tar cvzf ./dagger_engine_logs.tgz ./dagger_engine_logs
 
     - name: Upload logs to GitHub
-      id: upload-docker-logs
+      id: upload-dagger-engine-logs
       if: always()
       uses: actions/upload-artifact@v4
       with:
-        name: docker_logs.tgz
-        path: ./docker_logs.tgz
+        name: ${{ github.job }}_dagger_engine_logs.tgz
+        path: ./dagger_engine_logs.tgz
         retention-days: 7

.github/workflows/airbyte-ci-tests.yml

@@ -40,15 +40,14 @@ jobs:
             - airbyte-ci/connectors/ci_credentials/**
             - airbyte-ci/connectors/metadata_service/lib/**
             - airbyte-ci/connectors/metadata_service/orchestrator/**
+            - airbyte-cdk/python/**
             - airbyte-integrations/bases/connector-acceptance-test/**
 
   run-tests:
     needs: changes
-    # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
+    # We only run the Internal Poetry packages CI job if there are changes to the packages on a non-forked PR
     if: needs.changes.outputs.internal_poetry_packages == 'true'
-    #name: Internal Poetry packages CI
-    # To rename in a follow up PR
-    name: Run Airbyte CI tests
+    name: Internal Poetry packages CI
     runs-on: tooling-test-large
     permissions:
       pull-requests: read

.github/workflows/community_ci.yml

@@ -14,20 +14,131 @@ on:
   # It guarantees that the workflow logic is not altered by the PR.
   pull_request_target:
     types: [opened, synchronize]
-    # We only accept PRs touching connectors
-    paths:
-      - "airbyte-integrations/connectors/**"
-    branches:
-      - "master"
 
 jobs:
-  connectors_test:
-    name: Run connectors tests on fork
+  fail_on_protected_path_changes:
+    name: "Check fork do not change protected paths"
+    if: github.event.pull_request.head.repo.fork == true
+    runs-on: ubuntu-latest
+    permissions:
+      pull-requests: read
+    steps:
+      - name: Check for changes in protected paths
+        id: check_for_changes_in_protected_paths
+        uses: dorny/paths-filter@v2
+        with:
+          filters: |
+            protected_paths:
+              - '.github/**'
+              - 'airbyte-ci/**'
+
+      - name: Fail if changes in protected paths
+        if: steps.check_for_changes_in_protected_paths.outputs.protected_paths == 'true'
+        run: |
+          echo "The fork has changes in protected paths. This is not allowed."
+          exit 1
+
+  format_check:
+    # IMPORTANT: This name must match the require check name on the branch protection settings
+    name: "Check for formatting errors"
+    if: github.event.pull_request.head.repo.fork == true
+    environment: community-ci-auto
+    runs-on: community-tooling-test-small
+    needs: fail_on_protected_path_changes
+    timeout-minutes: 30
+    env:
+      MAIN_BRANCH_NAME: "master"
+    steps:
+      # This checkouts a fork which can contain untrusted code
+      # It's deemed safe as the formatter are not executing any checked out code
+      - name: Checkout fork
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 1
+
+      # This will sync the .github folder of the main repo with the fork
+      # This allows us to use up to date actions and CI logic from the main repo
+      - name: Pull .github folder and internal packages from main repository
+        id: pull_github_folder
+        run: |
+          git remote add main https://github.com/airbytehq/airbyte.git
+          git fetch main ${MAIN_BRANCH_NAME}
+          git checkout main/${MAIN_BRANCH_NAME} -- .github
+          git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
+
+      - name: Run airbyte-ci format check all
+        # This path refers to the fork .github folder.
+        # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: "pull_request"
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          subcommand: "format check all"
+          is_fork: "true"
+  connectors_early_ci:
+    name: Run connectors early CI on fork
+    if: github.event.pull_request.head.repo.fork == true
+    needs: fail_on_protected_path_changes
+    environment: community-ci-auto
+    runs-on: community-tooling-test-small
+    timeout-minutes: 10
+    env:
+      MAIN_BRANCH_NAME: "master"
+    permissions:
+      statuses: write
+    steps:
+      # This checkouts a fork which can contain untrusted code
+      # It's deemed safe as the static checks are not executing any checked out code
+      - name: Checkout fork
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
+          ref: ${{ github.event.pull_request.head.sha }}
+          fetch-depth: 1
+
+      # This will sync the .github folder of the main repo with the fork
+      # This allows us to use up to date actions and CI logic from the main repo
+      - name: Pull .github folder from main repository
+        id: pull_github_folder
+        run: |
+          git remote add main https://github.com/airbytehq/airbyte.git
+          git fetch main ${MAIN_BRANCH_NAME}
+          git checkout main/${MAIN_BRANCH_NAME} -- .github
+          git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
+
+      - name: Run airbyte-ci static checks and version increment checks on modified connectors
+        # This path refers to the fork .github folder.
+        # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: "pull_request"
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          subcommand: "connectors --modified test --only-step=qa_checks --only-step=version_inc_check --global-status-check-context='Connectors early CI checks' --global-status-check-description='Running early CI checks on connectors'"
+          is_fork: "true"
+          git_repo_url: ${{ github.event.pull_request.head.repo.clone_url }}
+          git_branch: ${{ github.head_ref }}
+          git_revision: ${{ github.event.pull_request.head.sha }}
+          github_token: ${{ github.token }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+      - name: Upload pipeline reports
+        id: upload-artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: early-ci-pipeline-reports
+          path: /home/runner/work/airbyte/airbyte/airbyte-ci/connectors/pipelines/pipeline_reports/airbyte-ci/connectors/test/pull_request/**/output.html
+          retention-days: 7
+
+  connectors_full_ci:
+    name: Run connectors full CI on fork
     if: github.event.pull_request.head.repo.fork == true
     # Deployment of jobs on the community-ci environment requires manual approval
     # This is something we set up in the GitHub environment settings:
     # https://github.com/airbytehq/airbyte/settings/environments/2091483613/edit
     # This is a safety measure to make sure the code running on our infrastructure has been reviewed before running on it
+    needs: fail_on_protected_path_changes
     environment: community-ci
     runs-on: community-connector-test-large
     timeout-minutes: 180 # 3 hours
@@ -47,13 +158,14 @@ jobs:
           fetch-depth: 1
 
       # This will sync the .github folder of the main repo with the fork
-      # This allows us to use up to date actions from the main repo
+      # This allows us to use up to date actions and CI logic from the main repo
       - name: Pull .github folder from main repository
         id: pull_github_folder
         run: |
           git remote add main https://github.com/airbytehq/airbyte.git
           git fetch main ${MAIN_BRANCH_NAME}
           git checkout main/${MAIN_BRANCH_NAME} -- .github
+          git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci
 
       - name: Run airbyte-ci connectors test
         # This path refers to the fork .github folder.

.github/workflows/connectors_tests.yml

@@ -63,7 +63,7 @@ jobs:
     # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
     # Forked PRs are handled by the community_ci.yml workflow
     # If the condition is not met the job will be skipped (it will not fail)
-    if: needs.changes.outputs.connectors == 'true' && github.event.pull_request.head.repo.fork != true
+    if: (github.event_name == 'pull_request' && needs.changes.outputs.connectors == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch'
     name: Connectors CI
     runs-on: connector-test-large
     timeout-minutes: 360 # 6 hours