From b9bcd5a95762022e3129fca97a3052cdebb23995 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Mon, 11 Mar 2024 08:51:30 +0100 Subject: [PATCH] Add Python 3.12 support (#36755) Finally after a number of dependency upgrades we seem to be able to upgrade to Python 3.12 (pending universal_pathlib 0.2.0 conversion) Several providers are excluded from being installed and wait for Python 3.12, but it should not block Airlfow's general 3.12 support. Co-authored-by: dirrao (cherry picked from commit 76dee0b5867580480d802765bad2dd61e3156bff) --- .github/workflows/ci.yml | 97 ++++++++------ .pre-commit-config.yaml | 2 +- Dockerfile | 2 +- Dockerfile.ci | 2 +- README.md | 2 +- airflow/__init__.py | 7 +- contributing-docs/07_local_virtualenv.rst | 4 +- dev/README_RELEASE_AIRFLOW.md | 2 +- dev/breeze/README.md | 2 +- dev/breeze/doc/ci/02_images.md | 2 +- dev/breeze/doc/ci/04_selective_checks.md | 29 +++-- dev/breeze/doc/ci/07_debugging.md | 17 ++- dev/breeze/doc/images/output-commands.svg | 2 +- .../doc/images/output_ci-image_build.svg | 4 +- .../doc/images/output_ci-image_build.txt | 2 +- .../doc/images/output_ci-image_pull.svg | 64 ++++----- .../doc/images/output_ci-image_pull.txt | 2 +- .../doc/images/output_ci-image_verify.svg | 4 +- .../doc/images/output_ci-image_verify.txt | 2 +- .../doc/images/output_k8s_build-k8s-image.svg | 56 ++++---- .../doc/images/output_k8s_build-k8s-image.txt | 2 +- .../images/output_k8s_configure-cluster.svg | 60 +++++---- .../images/output_k8s_configure-cluster.txt | 2 +- .../doc/images/output_k8s_create-cluster.svg | 4 +- .../doc/images/output_k8s_create-cluster.txt | 2 +- .../doc/images/output_k8s_delete-cluster.svg | 34 ++--- .../doc/images/output_k8s_delete-cluster.txt | 2 +- .../doc/images/output_k8s_deploy-airflow.svg | 76 +++++------ .../doc/images/output_k8s_deploy-airflow.txt | 2 +- dev/breeze/doc/images/output_k8s_k9s.svg | 32 +++-- dev/breeze/doc/images/output_k8s_k9s.txt | 2 +- dev/breeze/doc/images/output_k8s_logs.svg | 34 ++--- dev/breeze/doc/images/output_k8s_logs.txt | 2 +- .../images/output_k8s_run-complete-tests.svg | 70 +++++----- .../images/output_k8s_run-complete-tests.txt | 2 +- dev/breeze/doc/images/output_k8s_shell.svg | 42 +++--- dev/breeze/doc/images/output_k8s_shell.txt | 2 +- dev/breeze/doc/images/output_k8s_status.svg | 36 +++--- dev/breeze/doc/images/output_k8s_status.txt | 2 +- dev/breeze/doc/images/output_k8s_tests.svg | 70 +++++----- dev/breeze/doc/images/output_k8s_tests.txt | 2 +- .../images/output_k8s_upload-k8s-image.svg | 58 +++++---- .../images/output_k8s_upload-k8s-image.txt | 2 +- .../doc/images/output_prod-image_build.svg | 4 +- .../doc/images/output_prod-image_build.txt | 2 +- .../doc/images/output_prod-image_pull.svg | 64 ++++----- .../doc/images/output_prod-image_pull.txt | 2 +- .../doc/images/output_prod-image_verify.svg | 4 +- .../doc/images/output_prod-image_verify.txt | 2 +- ...elease-management_generate-constraints.svg | 4 +- ...elease-management_generate-constraints.txt | 2 +- ...management_generate-providers-metadata.svg | 2 +- ...management_generate-providers-metadata.txt | 2 +- ...e-management_install-provider-packages.svg | 122 +++++++++--------- ...e-management_install-provider-packages.txt | 2 +- ...release-management_release-prod-images.svg | 2 +- ...release-management_release-prod-images.txt | 2 +- ...se-management_verify-provider-packages.svg | 110 ++++++++-------- ...se-management_verify-provider-packages.txt | 2 +- .../output_sbom_build-all-airflow-images.svg | 2 +- .../output_sbom_build-all-airflow-images.txt | 2 +- ...t_sbom_generate-providers-requirements.svg | 2 +- ...t_sbom_generate-providers-requirements.txt | 2 +- .../output_sbom_update-sbom-information.svg | 2 +- .../output_sbom_update-sbom-information.txt | 2 +- dev/breeze/doc/images/output_setup_config.svg | 2 +- dev/breeze/doc/images/output_setup_config.txt | 2 +- dev/breeze/doc/images/output_shell.svg | 2 +- dev/breeze/doc/images/output_shell.txt | 2 +- .../doc/images/output_start-airflow.svg | 2 +- .../doc/images/output_start-airflow.txt | 2 +- .../doc/images/output_testing_db-tests.svg | 94 +++++++------- .../doc/images/output_testing_db-tests.txt | 2 +- .../output_testing_docker-compose-tests.svg | 2 +- .../output_testing_docker-compose-tests.txt | 2 +- .../output_testing_integration-tests.svg | 50 +++---- .../output_testing_integration-tests.txt | 2 +- .../images/output_testing_non-db-tests.svg | 90 +++++++------ .../images/output_testing_non-db-tests.txt | 2 +- .../doc/images/output_testing_tests.svg | 98 +++++++------- .../doc/images/output_testing_tests.txt | 2 +- dev/breeze/pyproject.toml | 2 +- .../commands/testing_commands.py | 6 + .../src/airflow_breeze/global_constants.py | 14 +- dev/breeze/src/airflow_breeze/utils/cdxgen.py | 1 + .../src/airflow_breeze/utils/packages.py | 15 +++ .../src/airflow_breeze/utils/run_tests.py | 40 +++++- .../airflow_breeze/utils/selective_checks.py | 66 +++++++--- dev/breeze/tests/test_cache.py | 4 +- dev/breeze/tests/test_packages.py | 2 +- .../tests/test_pytest_args_for_test_types.py | 17 ++- dev/breeze/tests/test_selective_checks.py | 97 +++++++++----- dev/retag_docker_images.py | 2 +- .../installation/prerequisites.rst | 2 +- docs/apache-airflow/start.rst | 2 +- .../docs_build/third_party_inventories.py | 4 +- generated/PYPI_README.md | 2 +- newsfragments/38025.significant.rst | 19 +++ pyproject.toml | 18 ++- ...tart_arm_instance_and_connect_to_docker.sh | 4 +- .../in_container/run_generate_constraints.py | 11 +- tests/always/test_example_dags.py | 31 ++++- tests/always/test_providers_manager.py | 25 +++- tests/always/test_secrets.py | 4 +- tests/providers/amazon/aws/hooks/test_emr.py | 4 + .../aws/transfers/test_azure_blob_to_s3.py | 2 +- .../fab/auth_manager/test_security.py | 8 +- .../cloud/transfers/test_cassandra_to_gcs.py | 90 +++++++------ .../utils/test_mlengine_prediction_summary.py | 55 ++++---- .../microsoft/azure/hooks/test_adx.py | 68 +++++----- .../microsoft/azure/hooks/test_cosmos.py | 8 +- .../azure/hooks/test_data_factory.py | 9 +- .../microsoft/azure/hooks/test_data_lake.py | 10 +- .../microsoft/azure/hooks/test_synapse.py | 9 +- .../microsoft/azure/hooks/test_wasb.py | 8 +- .../providers/papermill/hooks/test_kernel.py | 3 +- .../papermill/operators/test_papermill.py | 60 ++++++--- tests/serialization/test_dag_serialization.py | 17 ++- tests/www/api/experimental/test_endpoints.py | 5 + 119 files changed, 1342 insertions(+), 942 deletions(-) create mode 100644 newsfragments/38025.significant.rst diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 041a83c420d83..d3d69b22995dd 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -129,6 +129,7 @@ jobs: is-arm-runner: ${{ steps.selective-checks.outputs.is-arm-runner }} is-vm-runner: ${{ steps.selective-checks.outputs.is-vm-runner }} is-k8s-runner: ${{ steps.selective-checks.outputs.is-k8s-runner }} + latest-versions-only: ${{ steps.selective-checks.outputs.latest-versions-only }} chicken-egg-providers: ${{ steps.selective-checks.outputs.chicken-egg-providers }} has-migrations: ${{ steps.selective-checks.outputs.has-migrations }} source-head-repo: ${{ steps.source-run-info.outputs.source-head-repo }} @@ -208,22 +209,22 @@ jobs: matrix.platform == 'linux/arm64' && needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' - name: "Push CI cache ${{ matrix.platform }}" - run: > - breeze ci-image build - --builder airflow_cache - --prepare-buildx-cache - --run-in-parallel - --platform ${{ matrix.platform }} + run: | + for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do + breeze ci-image build --builder airflow_cache --prepare-buildx-cache \ + --python ${PYTHON} --platform ${{ matrix.platform }} + done env: - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} COMMIT_SHA: ${{ github.sha }} + PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} if: needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' - name: "Push CI latest image ${{ matrix.platform }}" - run: > - breeze ci-image build - --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} + run: | + for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do + breeze ci-image build --tag-as-latest --push --run-in-parallel \ + --python ${PYTHON} --platform ${{ matrix.platform }} + done env: - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} COMMIT_SHA: ${{ github.sha }} # We only push "amd" image as it is really only needed for any kind of automated builds in CI # and currently there is not an easy way to make multi-platform image from two separate builds @@ -252,6 +253,8 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" UPGRADE_TO_NEWER_DEPENDENCIES: false PLATFORM: "linux/amd64" + PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" + PYTHON_VERSION: ${{needs.build-info.outputs.default-python-version}} if: > needs.build-info.outputs.canary-run == 'true' && needs.build-info.outputs.default-branch == 'main' @@ -267,6 +270,7 @@ jobs: - name: "Check that image builds quickly" run: breeze shell --max-time 120 + build-ci-images: strategy: fail-fast: true @@ -566,7 +570,9 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" - if: needs.build-info.outputs.basic-checks-only == 'false' + if: > + needs.build-info.outputs.basic-checks-only == 'false' && + needs.build-info.outputs.latest-versions-only != 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -604,7 +610,9 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" - if: needs.build-info.outputs.canary-run == 'true' + if: > + needs.build-info.outputs.canary-run == 'true' && + needs.build-info.outputs.latest-versions-only != 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -666,6 +674,7 @@ jobs: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" UPGRADE_TO_NEWER_DEPENDENCIES: "${{ needs.build-info.outputs.upgrade-to-newer-dependencies }}" + if: needs.build-info.outputs.latest-versions-only != 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -760,7 +769,9 @@ jobs: name: "Build docs" runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} needs: [build-info, wait-for-ci-images] - if: needs.build-info.outputs.docs-build == 'true' + if: > + needs.build-info.outputs.docs-build == 'true' && + needs.build-info.outputs.latest-versions-only != 'true' env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" @@ -824,7 +835,9 @@ jobs: name: "Spellcheck docs" runs-on: ${{fromJSON(needs.build-info.outputs.runs-on)}} needs: [build-info, wait-for-ci-images] - if: needs.build-info.outputs.docs-build == 'true' + if: > + needs.build-info.outputs.docs-build == 'true' && + needs.build-info.outputs.latest-versions-only != 'true' env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" @@ -858,7 +871,9 @@ jobs: env: RUNS_ON: "${{needs.build-info.outputs.runs-on}}" PYTHON_MAJOR_MINOR_VERSION: "${{needs.build-info.outputs.default-python-version}}" - if: needs.build-info.outputs.skip-provider-tests != 'true' + if: > + needs.build-info.outputs.skip-provider-tests != 'true' && + needs.build-info.outputs.latest-versions-only != 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -1085,7 +1100,9 @@ jobs: JOB_ID: "helm-tests" USE_XDIST: "true" if: > - needs.build-info.outputs.needs-helm-tests == 'true' && needs.build-info.outputs.default-branch == 'main' + needs.build-info.outputs.needs-helm-tests == 'true' && + needs.build-info.outputs.default-branch == 'main' && + needs.build-info.outputs.latest-versions-only != 'true' steps: - name: Cleanup repo run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" @@ -1385,7 +1402,9 @@ jobs: JOB_ID: > pendulum-2-${{needs.build-info.outputs.default-python-version}}- ${{needs.build-info.outputs.default-postgres-version}} - if: needs.build-info.outputs.run-tests == 'true' + if: > + needs.build-info.outputs.run-tests == 'true' && + needs.build-info.outputs.latest-versions-only != 'true' steps: - name: Cleanup repo shell: bash @@ -1796,7 +1815,9 @@ jobs: JOB_ID: "no-db-pendulum-2-${{needs.build-info.outputs.default-python-version}}" ENABLE_COVERAGE: "${{needs.build-info.outputs.run-coverage}}" DOWNGRADE_PENDULUM: "true" - if: needs.build-info.outputs.run-tests == 'true' + if: > + needs.build-info.outputs.run-tests == 'true' && + needs.build-info.outputs.latest-versions-only != 'true' steps: - name: Cleanup repo shell: bash @@ -2210,19 +2231,19 @@ jobs: run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh if: matrix.platform == 'linux/arm64' - name: "Push CI cache ${{ matrix.platform }}" - run: > - breeze ci-image build - --builder airflow_cache - --prepare-buildx-cache - --run-in-parallel - --platform ${{ matrix.platform }} + run: | + for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do + breeze ci-image build --builder airflow_cache --prepare-buildx-cache \ + --python ${PYTHON} --platform ${{ matrix.platform }} + done env: - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} COMMIT_SHA: ${{ github.sha }} - name: "Push CI latest image ${{ matrix.platform }}" - run: > - breeze ci-image build - --tag-as-latest --push --run-in-parallel --platform ${{ matrix.platform }} + run: | + for PYTHON in ${{needs.build-info.outputs.python-versions-list-as-string}}; do + breeze ci-image build --tag-as-latest --push \ + --python ${PYTHON} --platform ${{ matrix.platform }} + done env: DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} COMMIT_SHA: ${{ github.sha }} @@ -2231,8 +2252,6 @@ jobs: run: cp -v --no-preserve=mode,ownership ./dist/*.whl ./docker-context-files - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" run: | - # Do not run parallel builds here as they often fail due to github token expiry issues similar to - # those described in https://github.com/moby/buildkit/issues/2367 for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do breeze prod-image build --builder airflow_cache --install-packages-from-context \ --airflow-constraints-mode constraints-source-providers --prepare-buildx-cache \ @@ -2241,9 +2260,11 @@ jobs: env: COMMIT_SHA: ${{ github.sha }} - name: "Push PROD latest image ${{ matrix.platform }}" - run: > - breeze prod-image build --tag-as-latest --install-packages-from-context - --push --run-in-parallel --platform ${{ matrix.platform }} + run: | + for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do + breeze prod-image build --tag-as-latest --install-packages-from-context \ + --push --run-in-parallel --python ${PYTHON} --platform ${{ matrix.platform }} + done env: DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} COMMIT_SHA: ${{ github.sha }} @@ -2298,13 +2319,13 @@ jobs: - name: > Build CI ARM images ${{ needs.build-info.outputs.image-tag }} ${{needs.build-info.outputs.all-python-versions-list-as-string}}:${{env.IMAGE_TAG}} - run: > - breeze ci-image build --run-in-parallel --builder airflow_cache --platform "linux/arm64" + run: | + for python in ${{needs.build-info.outputs.python-versions-list-as-string}}; do + breeze ci-image build --builder airflow_cache --platform "linux/arm64" --python ${python} + done env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} - PYTHON_VERSIONS: ${{needs.build-info.outputs.all-python-versions-list-as-string}} - DEBUG_RESOURCES: ${{needs.build-info.outputs.debug-resources}} COMMIT_SHA: ${{ github.sha }} - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 5a1d22e71f177..68d7c688a9db3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -704,7 +704,7 @@ repos: name: Verify usage of Airflow deprecation classes in core entry: category=DeprecationWarning|category=PendingDeprecationWarning files: \.py$ - exclude: ^airflow/configuration\.py$|^airflow/providers|^scripts/in_container/verify_providers\.py$ + exclude: ^airflow/configuration\.py$|^airflow/providers|^scripts/in_container/verify_providers\.py$|^tests/.*$ pass_filenames: true - id: check-provide-create-sessions-imports language: pygrep diff --git a/Dockerfile b/Dockerfile index 78a4c16407181..f6adeabf36a70 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1528,7 +1528,7 @@ ARG USE_CONSTRAINTS_FOR_CONTEXT_PACKAGES="false" # By changing the epoch we can force reinstalling Airflow and pip all dependencies # It can also be overwritten manually by setting the AIRFLOW_CI_BUILD_EPOCH environment variable. -ARG AIRFLOW_CI_BUILD_EPOCH="10" +ARG AIRFLOW_CI_BUILD_EPOCH="11" ENV AIRFLOW_CI_BUILD_EPOCH=${AIRFLOW_CI_BUILD_EPOCH} diff --git a/Dockerfile.ci b/Dockerfile.ci index 8779c8383c793..5506c56c575c9 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1108,7 +1108,7 @@ ARG PYTHON_BASE_IMAGE ARG AIRFLOW_IMAGE_REPOSITORY="https://github.com/apache/airflow" # By increasing this number we can do force build of all dependencies -ARG DEPENDENCIES_EPOCH_NUMBER="10" +ARG DEPENDENCIES_EPOCH_NUMBER="11" # Make sure noninteractive debian install is used and language variables set ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ diff --git a/README.md b/README.md index 50e558d712980..7f726cfd4583b 100644 --- a/README.md +++ b/README.md @@ -100,7 +100,7 @@ Apache Airflow is tested with: | | Main version (dev) | Stable version (2.8.2) | |-------------|------------------------------|------------------------| -| Python | 3.8, 3.9, 3.10, 3.11 | 3.8, 3.9, 3.10, 3.11 | +| Python | 3.8, 3.9, 3.10, 3.11, 3.12 | 3.8, 3.9, 3.10, 3.11 | | Platform | AMD64/ARM64(\*) | AMD64/ARM64(\*) | | Kubernetes | 1.25, 1.26, 1.27, 1.28, 1.29 | 1.25, 1.26, 1.27, 1.28 | | PostgreSQL | 12, 13, 14, 15, 16 | 12, 13, 14, 15, 16 | diff --git a/airflow/__init__.py b/airflow/__init__.py index 234f6d349389d..6b06293876e09 100644 --- a/airflow/__init__.py +++ b/airflow/__init__.py @@ -39,7 +39,12 @@ # configuration is therefore initted early here, simply by importing it. from airflow import configuration, settings -__all__ = ["__version__", "DAG", "Dataset", "XComArg"] +__all__ = [ + "__version__", + "DAG", + "Dataset", + "XComArg", +] # Make `airflow` a namespace package, supporting installing # airflow.providers.* in different locations (i.e. one in site, and one in user diff --git a/contributing-docs/07_local_virtualenv.rst b/contributing-docs/07_local_virtualenv.rst index 6d6406cce1ffe..1ded48c0fa460 100644 --- a/contributing-docs/07_local_virtualenv.rst +++ b/contributing-docs/07_local_virtualenv.rst @@ -37,7 +37,7 @@ Required Software Packages Use system-level package managers like yum, apt-get for Linux, or Homebrew for macOS to install required software packages: -* Python (One of: 3.8, 3.9, 3.10, 3.11) +* Python (One of: 3.8, 3.9, 3.10, 3.11, 3.12) * MySQL 5.7+ * libxml * helm (only for helm chart tests) @@ -186,6 +186,8 @@ This is what it shows currently: +-------------+---------+----------+---------------------------------------------------------------+ | airflow-311 | virtual | devel | Environment with Python 3.11 | +-------------+---------+----------+---------------------------------------------------------------+ +| airflow-312 | virtual | devel | Environment with Python 3.12 | ++-------------+---------+----------+---------------------------------------------------------------+ The default env (if you have not used one explicitly) is ``default`` and it is a Python 3.8 virtualenv for maximum compatibility with ``devel`` extra installed - this devel extra contains the minimum set diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index 4a929b8e0bdfb..8a682cbeb4805 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -822,7 +822,7 @@ the older branches, you should set the "skip" field to true. ## Verify production images ```shell script -for PYTHON in 3.8 3.9 3.10 3.11 +for PYTHON in 3.8 3.9 3.10 3.11 3.12 do docker pull apache/airflow:${VERSION}-python${PYTHON} breeze prod-image verify --image-name apache/airflow:${VERSION}-python${PYTHON} diff --git a/dev/breeze/README.md b/dev/breeze/README.md index a6e6419c3625f..9459f40f378f9 100644 --- a/dev/breeze/README.md +++ b/dev/breeze/README.md @@ -66,6 +66,6 @@ PLEASE DO NOT MODIFY THE HASH BELOW! IT IS AUTOMATICALLY UPDATED BY PRE-COMMIT. --------------------------------------------------------------------------------------------------------- -Package config hash: e2db123fd25e40b515520fb9f6ed32a601fe85e6a22b9845343bc5c81e5785b472527ed3bf6d07521c512d2222f99877a1ce1911ac504a3a6af7b7866aa674cc +Package config hash: 0c6255210f3c20a29aa24405412399cf3f6ff897658f13a0fb7628b4888d6bfe99f49bdb2aa68d9045e14179d4be33a02543c12e394395931024522431bf5dec --------------------------------------------------------------------------------------------------------- diff --git a/dev/breeze/doc/ci/02_images.md b/dev/breeze/doc/ci/02_images.md index 7807d94b1e242..253331e48b7c3 100644 --- a/dev/breeze/doc/ci/02_images.md +++ b/dev/breeze/doc/ci/02_images.md @@ -574,7 +574,7 @@ percent-encoded when you access them via UI (/ = %2F) - \ might be either "main" or "v2-\*-test" - \ - Python version (Major + Minor).Should be one of \["3.8", - "3.9", "3.10", "3.11"\]. + "3.9", "3.10", "3.11", "3.12" \]. - \ - full-length SHA of commit either from the tip of the branch (for pushes/schedule) or commit from the tip of the branch used for the PR. diff --git a/dev/breeze/doc/ci/04_selective_checks.md b/dev/breeze/doc/ci/04_selective_checks.md index 02e137c82839b..e53d7c0448ed4 100644 --- a/dev/breeze/doc/ci/04_selective_checks.md +++ b/dev/breeze/doc/ci/04_selective_checks.md @@ -264,24 +264,33 @@ Also, for most of the jobs, committer builds by default use "Self-hosted" runner builds use "Public" runners. For committers, this can be overridden by setting the `use public runners` label in the PR. -If you are testing ci workflow changes and want to limit the number of matrix combinations generated by +If you are testing CI workflow changes and want to limit the number of matrix combinations generated by the jobs - you can set `default versions only` label in the PR. This will limit the number of versions used in the matrix to the default ones (default Python version and default Kubernetes version). +If you are testing CI workflow changes and want to limit the number of matrix combinations generated by +the jobs - you can also set `latest versions only` label in the PR. This will limit the number of versions +used in the matrix to the latest ones (latest Python version and latest Kubernetes version). + +You can also disable cache if you want to make sure your tests will run with image that does not have +left-over package installed from the past cached image - by setting `disable image cache` label in the PR. + ## PR labels As mentioned below, you can influence the outputs of selected checks by setting labels to the PR. Here is am overview of possible labels and their meaning: -| Label | Affected outputs | Meaning | -|-------------------------------|------------------------------------------|--------------------------------------------------------------------------------------------------------| -| canary | is-canary-run | If set, the PR run from apache/airflow repo behaves as `canary` run (can only be run by maintainer). | -| debug ci resources | debug-ci-resources | If set, then debugging resources is enabled during parallel tests and you can see them in the output. | -| default versions only | python-versions-*, kubernetes-versions-* | If set, the number of Python and Kubernetes versions used by the build will be limitd to default ones. | -| full tests needed | full-tests-needed | Run complete set of tests, including all Python all DB versions, and all test types. | -| non committer build | is-committer-build | If set then even for non-committer builds, the scripts used for images are used from target branch. | -| upgrade to newer dependencies | upgrade-to-newer-dependencies | If set then dependencies in the CI image build are upgraded to the newer ones. | -| use public runners | runs-on | Force using public runners even for Committer runs. | +| Label | Affected outputs | Meaning | +|-------------------------------|------------------------------------------|-------------------------------------------------------------------------------------------------------------| +| canary | is-canary-run | If set, the PR run from apache/airflow repo behaves as `canary` run (can only be run by maintainer). | +| debug ci resources | debug-ci-resources | If set, then debugging resources is enabled during parallel tests and you can see them in the output. | +| default versions only | python-versions-*, kubernetes-versions-* | If set, the number of Python and Kubernetes versions used by the build will be limited to the default ones. | +| disable image cache | | If set, the number of Python and Kubernetes versions used by the build will be limited to the latest ones. | +| latest versions only | python-versions-*, kubernetes-versions-* | If set, the number of Python and Kubernetes versions used by the build will be limited to the latest ones. | +| full tests needed | full-tests-needed | Run complete set of tests, including all Python all DB versions, and all test types. | +| non committer build | is-committer-build | If set then even for non-committer builds, the scripts used for images are used from target branch. | +| upgrade to newer dependencies | upgrade-to-newer-dependencies | If set then dependencies in the CI image build are upgraded to the newer ones. | +| use public runners | runs-on | Force using public runners even for Committer runs. | ----- diff --git a/dev/breeze/doc/ci/07_debugging.md b/dev/breeze/doc/ci/07_debugging.md index 94ffd99329535..0516a95d2b91d 100644 --- a/dev/breeze/doc/ci/07_debugging.md +++ b/dev/breeze/doc/ci/07_debugging.md @@ -54,10 +54,11 @@ maintainer. rather than using `Build images` workflow and use the same breeze version for building image and testing - When you want to test changes to workflows and CI scripts you can set - `default versions only` label to the PR. This will make the PR run - using the default versions of Python and Kubernetes. This is useful - when you want to test changes to the CI scripts and workflows and you - want to use far less resources than the full tests. + `default versions only` label to the PR or `latest versions only`. + This will make the PR run using the default (or latest) versions of + Python and Kubernetes. This is useful when you want to test changes to + the CI scripts and workflows and you want to use far less resources + than the full tests. - When you want to test changes to `build-images.yml` workflow you should push your branch as `main` branch in your local fork. This will run changed `build-images.yml` workflow as it will be in `main` branch @@ -67,9 +68,11 @@ maintainer. in `build-images.yml` workflow, but the workflow is run using the `main` version. Setting `non committer build` label will make your PR run using the main version of the scripts and the workflow -- When you want to test how changes in your workflow affect `canary` run, - as maintainer, you should push your PR to `apache` repository not to your - fork and set `canary` label to the PR +- When you are a committer want to test how changes in your workflow affect + `canary` run, as maintainer, you should push your PR to `apache` repository + not to your fork and set `canary` label to the PR +- When you are a committer and want to test if the tests are passing if the + image is freshly built without cache, you can set `disable image cache` label. ----- diff --git a/dev/breeze/doc/images/output-commands.svg b/dev/breeze/doc/images/output-commands.svg index f0886818f2544..d1befe15946fe 100644 --- a/dev/breeze/doc/images/output-commands.svg +++ b/dev/breeze/doc/images/output-commands.svg @@ -281,7 +281,7 @@ ╭─ Execution mode ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ +│(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ │--integrationIntegration(s) to enable when running (can be more than one).                       │ │(all | all-testable | cassandra | celery | kafka | kerberos | mongo | openlineage | │ diff --git a/dev/breeze/doc/images/output_ci-image_build.svg b/dev/breeze/doc/images/output_ci-image_build.svg index 48b98e5904acc..7e9ffae1376c0 100644 --- a/dev/breeze/doc/images/output_ci-image_build.svg +++ b/dev/breeze/doc/images/output_ci-image_build.svg @@ -346,7 +346,7 @@ ╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ │--python-pPython major/minor version used in Airflow image for images.│ -│(>3.8< | 3.9 | 3.10 | 3.11)                                 │ +│(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ │[default: 3.8]                                              │ │--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ │--upgrade-on-failure/--no-upgrade-on-failureWhen set, attempt to run upgrade to newer dependencies when       │ @@ -374,7 +374,7 @@ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ +│[default: 3.8 3.9 3.10 3.11 3.12]                                             │ │--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_ci-image_build.txt b/dev/breeze/doc/images/output_ci-image_build.txt index d32ec3ed79812..782ff33f9cf83 100644 --- a/dev/breeze/doc/images/output_ci-image_build.txt +++ b/dev/breeze/doc/images/output_ci-image_build.txt @@ -1 +1 @@ -775924a9beade1c361b7b0d127e21321 +eeb71cb74afe5c0a59ea3236626c79e6 diff --git a/dev/breeze/doc/images/output_ci-image_pull.svg b/dev/breeze/doc/images/output_ci-image_pull.svg index fd74f3d396228..78616b54218f9 100644 --- a/dev/breeze/doc/images/output_ci-image_pull.svg +++ b/dev/breeze/doc/images/output_ci-image_pull.svg @@ -1,4 +1,4 @@ - + ╭─ Pull image flags ───────────────────────────────────────────────────────────────────────────────────────────────────╮ │--image-tag-tTag of the image which is used to pull the image.(TEXT)[default: latest]│ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--verifyVerify image.│ -│--wait-for-imageWait until image is available.│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ -│or pull image with --image-tag.                                                               │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ +│[default: 3.8]                                              │ +│--verifyVerify image.│ +│--wait-for-imageWait until image is available.│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful when you build│ +│or pull image with --image-tag.                                                               │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel running ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.8 3.9 3.10 3.11 3.12]                                             │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Github authentication ──────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_ci-image_pull.txt b/dev/breeze/doc/images/output_ci-image_pull.txt index 3eb380722d54f..c916be1bf34eb 100644 --- a/dev/breeze/doc/images/output_ci-image_pull.txt +++ b/dev/breeze/doc/images/output_ci-image_pull.txt @@ -1 +1 @@ -f9248c6026da61fe0acdb5d8f37b20da +3c950cd0e358661163e52c9f3726aee8 diff --git a/dev/breeze/doc/images/output_ci-image_verify.svg b/dev/breeze/doc/images/output_ci-image_verify.svg index cb247415e90b7..2a0431e876c4d 100644 --- a/dev/breeze/doc/images/output_ci-image_verify.svg +++ b/dev/breeze/doc/images/output_ci-image_verify.svg @@ -157,7 +157,7 @@ ╭─ Verify image flags ─────────────────────────────────────────────────────────────────────────────────────────────────╮ │--image-name-nName of the image to verify (overrides --python and --image-tag).(TEXT)│ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ +│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)│ │[default: 3.8]                                              │ │--image-tag-tTag of the image when verifying it.(TEXT)[default: latest]│ │--pullPull image is missing before attempting to verify it.│ @@ -168,7 +168,7 @@ │(INTEGER RANGE)                                                            │ │[default: 4; 1<=x<=8]                                                      │ │--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ +│[default: 3.8 3.9 3.10 3.11 3.12]                                             │ │--skip-cleanupSkip cleanup of temporary files created during parallel run.│ │--debug-resourcesWhether to show resource information while running in parallel.│ │--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ diff --git a/dev/breeze/doc/images/output_ci-image_verify.txt b/dev/breeze/doc/images/output_ci-image_verify.txt index f454443f6888e..bad4f7d8a8bae 100644 --- a/dev/breeze/doc/images/output_ci-image_verify.txt +++ b/dev/breeze/doc/images/output_ci-image_verify.txt @@ -1 +1 @@ -707a149f99bd49d37be5b8d0db844d69 +ec059842b7c4c41db33a6362a361b0ef diff --git a/dev/breeze/doc/images/output_k8s_build-k8s-image.svg b/dev/breeze/doc/images/output_k8s_build-k8s-image.svg index c54802ce41769..fb948d36e32b5 100644 --- a/dev/breeze/doc/images/output_k8s_build-k8s-image.svg +++ b/dev/breeze/doc/images/output_k8s_build-k8s-image.svg @@ -1,4 +1,4 @@ - + Build k8s-ready airflow image (optionally all images in parallel). ╭─ Build image flags ──────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.(>3.8< | 3.9 | 3.10 | 3.11)│ -│[default: 3.8]                                              │ -│--image-tag-tImage tag used to build K8S image from.(TEXT)[default: latest]│ -│--rebuild-base-imageRebuilds base Airflow image before building K8S image.│ -│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.│ -│(INTEGER RANGE)                                                            │ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.8 3.9 3.10 3.11]                                                  │ -│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ -│--debug-resourcesWhether to show resource information while running in parallel.│ -│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.8< | 3.9 | 3.10 | 3.11 | 3.12)                          │ +│[default: 3.8]                                              │ +│--image-tag-tImage tag used to build K8S image from.(TEXT)[default: latest]│ +│--rebuild-base-imageRebuilds base Airflow image before building K8S image.│ +│--use-uv/--no-use-uvUse uv instead of pip as packaging tool to build the image.[default: use-uv]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Parallel options ───────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of parameters.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.│ +│(INTEGER RANGE)                                                            │ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.8 3.9 3.10 3.11 3.12]                                             │ +│--skip-cleanupSkip cleanup of temporary files created during parallel run.│ +│--debug-resourcesWhether to show resource information while running in parallel.│ +│--include-success-outputsWhether to include outputs of successful parallel runs (skipped by default).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_k8s_build-k8s-image.txt b/dev/breeze/doc/images/output_k8s_build-k8s-image.txt index c375b65d68cf8..8a35763e568d3 100644 --- a/dev/breeze/doc/images/output_k8s_build-k8s-image.txt +++ b/dev/breeze/doc/images/output_k8s_build-k8s-image.txt @@ -1 +1 @@ -2910945bcb00e309668da7a64773b859 +1739395599162dcc262156a26dd9b2e7 diff --git a/dev/breeze/doc/images/output_k8s_configure-cluster.svg b/dev/breeze/doc/images/output_k8s_configure-cluster.svg index 30f1ca16747e5..bb17e8f829d9a 100644 --- a/dev/breeze/doc/images/output_k8s_configure-cluster.svg +++ b/dev/breeze/doc/images/output_k8s_configure-cluster.svg @@ -1,4 +1,4 @@ - +