refactor!(bzlmod): introduce pypi.install extension

CHANGELOG.md

@@ -27,6 +27,10 @@ A brief description of the categories of changes:
 ### Changed
 * (toolchains) `py_runtime.implementation_name` now defaults to `cpython`
   (previously it defaulted to None).
+* (bzlmod) **BREAKING** The `experimental_index_url` feature has been removed
+  from `pip.parse` and a new extension (`pypi.install`) has been created in
+  order to ensure that `MODULE.bazel.lock` remains working for others.
+  Fixes [#2268](https://github.com/bazelbuild/rules_python/issues/2268).
 
 ### Fixed
 * (bzlmod) The `python.override(minor_mapping)` now merges the default and the

MODULE.bazel

@@ -54,8 +54,8 @@ register_toolchains("@pythons_hub//:all")
 #####################
 # Install twine for our own runfiles wheel publishing and allow bzlmod users to use it.
 
-pip = use_extension("//python/private/pypi:pip.bzl", "pip_internal")
-pip.parse(
+pypi = use_extension("//python/private/pypi:pypi.bzl", "pypi")
+pypi.install(
     hub_name = "rules_python_publish_deps",
     python_version = "3.11",
     requirements_by_platform = {
@@ -64,7 +64,7 @@ pip.parse(
         "//tools/publish:requirements_windows.txt": "windows_*",
     },
 )
-use_repo(pip, "rules_python_publish_deps")
+use_repo(pypi, "rules_python_publish_deps")
 
 # ===== DEV ONLY DEPS AND SETUP BELOW HERE =====
 bazel_dep(name = "stardoc", version = "0.6.2", dev_dependency = True, repo_name = "io_bazel_stardoc")
@@ -85,22 +85,22 @@ dev_python.override(
     register_all_versions = True,
 )
 
-dev_pip = use_extension(
-    "//python/private/pypi:pip.bzl",
-    "pip_internal",
+dev_pypi = use_extension(
+    "//python/private/pypi:pypi.bzl",
+    "pypi",
     dev_dependency = True,
 )
-dev_pip.parse(
+dev_pypi.install(
     hub_name = "dev_pip",
     python_version = "3.11",
     requirements_lock = "//docs:requirements.txt",
 )
-dev_pip.parse(
+dev_pypi.install(
     hub_name = "pypiserver",
     python_version = "3.11",
     requirements_lock = "//examples/wheel:requirements_server.txt",
 )
-use_repo(dev_pip, "dev_pip", "pypiserver")
+use_repo(dev_pypi, "dev_pip", "pypiserver")
 
 # Bazel integration test setup below
 

docs/BUILD.bazel

@@ -109,6 +109,7 @@ sphinx_stardocs(
     ] if IS_BAZEL_7_OR_HIGHER else []) + ([
         # This depends on @pythons_hub, which is only created under bzlmod,
         "//python/extensions:pip_bzl",
+        "//python/extensions:pypi_bzl",
     ] if IS_BAZEL_7_OR_HIGHER and BZLMOD_ENABLED else []),
     prefix = "api/rules_python/",
     tags = ["docs"],

docs/pypi-dependencies.md

@@ -13,10 +13,10 @@ Using PyPI packages (aka "pip install") involves two main steps.
 
 ### Using bzlmod
 
-To add pip dependencies to your `MODULE.bazel` file, use the `pip.parse`
-extension, and call it to create the central external repo and individual wheel
-external repos. Include in the `MODULE.bazel` the toolchain extension as shown
-in the first bzlmod example above.
+To add pip dependencies to your `MODULE.bazel` file, use the {bzl:obj}`pip.parse` or 
+{bzl:obj}`pypi.install` extension, and call it to create the central external
+repo and individual wheel external repos. Include in the `MODULE.bazel` the
+toolchain extension as shown in the first bzlmod example above.
 
 ```starlark
 pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
@@ -27,6 +27,7 @@ pip.parse(
 )
 use_repo(pip, "my_deps")
 ```
+
 For more documentation, including how the rules can update/create a requirements
 file, see the bzlmod examples under the {gh-path}`examples` folder or the documentation
 for the {obj}`@rules_python//python/extensions:pip.bzl` extension.
@@ -44,6 +45,78 @@ hermetic host python interpreter on this platform. Linux and OSX users should se
 difference.
 ```
 
+(bazel-downloader)=
+#### Using `pypi.install` extension
+
+:::{warning}
+The APIs here have been well tested but may still change in backwards
+incompatible ways if major design flaws are found.
+:::
+
+`pypi.install` provides almost a drop-in replacement for the `pip.parse` extension:
+```starlark
+pypi = use_extension("@rules_python//python/extensions:pypi.bzl", "pypi")
+pypi.install(
+    hub_name = "my_deps",
+    python_version = "3.11",
+    requirements_lock = "//:requirements_lock_3_11.txt",
+)
+use_repo(pypi, "my_deps")
+```
+
+The `bzlmod` `pypi.install` call supports pulling information from `PyPI` (or a
+compatible mirror) and it will ensure that the [bazel
+downloader][bazel_downloader] is used for downloading the wheels or the sdists
+in the requirements file. This allows the users to use the [credential
+helper](#credential-helper) to authenticate with the mirror and it also ensures
+that the distribution downloads are cached.
+It also avoids using `pip` altogether and results in much faster dependency
+fetching.
+
+See the {gh-path}`examples/bzlmod/MODULE.bazel` example for more info.
+
+:::{warning}
+For now there is a limitation that the lock
+files consumed by the `pypi.install` extension must have hashes, similarly to
+all our examples. Hence, the `requirements.txt` files produced by `pip lock` or
+by `uv pip compile` but without generating hashes will fail.
+:::
+
+Note, when using this feature during the `pip` extension evaluation you will see the accessed indexes similar to below:
+```console
+Loading: 0 packages loaded
+    currently loading: docs/
+    Fetching module extension pip in @@//python/extensions:pypi.bzl; starting
+    Fetching https://pypi.org/simple/twine/
+```
+
+This does not mean that `rules_python` is fetching the wheels eagerly, but it
+rather means that it is calling the PyPI server to get the Simple API response
+to get the list of all available source and wheel distributions. Once it has
+got all of the available distributions, it will select the right ones depending
+on the `sha256` values in your `requirements_lock.txt` file. The compatible
+distribution URLs will be then written to the `MODULE.bazel.lock` file. Currently
+users wishing to use the lock file with `rules_python` with this feature have
+to set an environment variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will
+become default in the next release.
+
+Fetching the distribution information from the PyPI allows `rules_python` to
+know which `whl` should be used on which target platform and it will determine
+that by parsing the `whl` filename based on [PEP600], [PEP656] standards. This
+allows the user to configure the behaviour by using the following publicly
+available flags:
+* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant.
+* {obj}`--@rules_python//python/config_settings:pip_whl` for selecting `whl` distribution preference.
+* {obj}`--@rules_python//python/config_settings:pip_whl_osx_arch` for selecting MacOS wheel preference.
+* {obj}`--@rules_python//python/config_settings:pip_whl_glibc_version` for selecting the GLIBC version compatibility.
+* {obj}`--@rules_python//python/config_settings:pip_whl_muslc_version` for selecting the musl version compatibility.
+* {obj}`--@rules_python//python/config_settings:pip_whl_osx_version` for selecting MacOS version compatibility.
+
+[bazel_downloader]: https://bazel.build/rules/lib/builtins/repository_ctx#download
+[pep600]: https://peps.python.org/pep-0600/
+[pep656]: https://peps.python.org/pep-0656/
+
+
 ### Using a WORKSPACE file
 
 To add pip dependencies to your `WORKSPACE`, load the `pip_parse` function and
@@ -134,6 +207,13 @@ re-executed to pick up a non-hermetic change to your environment (e.g., updating
 your system `python` interpreter), you can force it to re-execute by running
 `bazel sync --only [pip_parse name]`.
 
+:::{note}
+The same is not necessarily true for the `pypi.install` bazel extension as it is
+not relying on the built in interpreter environment information and yields the same
+set of dependencies irrespective of the host machine and the python interpreter used
+to pull the Python dependencies.
+:::
+
 {#using-third-party-packages}
 ## Using third party packages as dependencies
 
@@ -306,55 +386,6 @@ leg of the dependency manually. For instance by making
 `apache-airflow-providers-postgres` not explicitly depend on `apache-airflow` or
 perhaps `apache-airflow-providers-common-sql`.
 
-
-(bazel-downloader)=
-### Bazel downloader and multi-platform wheel hub repository.
-
-The `bzlmod` `pip.parse` call supports pulling information from `PyPI` (or a
-compatible mirror) and it will ensure that the [bazel
-downloader][bazel_downloader] is used for downloading the wheels. This allows
-the users to use the [credential helper](#credential-helper) to authenticate
-with the mirror and it also ensures that the distribution downloads are cached.
-It also avoids using `pip` altogether and results in much faster dependency
-fetching.
-
-This can be enabled by `experimental_index_url` and related flags as shown in
-the {gh-path}`examples/bzlmod/MODULE.bazel` example.
-
-When using this feature during the `pip` extension evaluation you will see the accessed indexes similar to below:
-```console
-Loading: 0 packages loaded
-    currently loading: docs/
-    Fetching module extension pip in @@//python/extensions:pip.bzl; starting
-    Fetching https://pypi.org/simple/twine/
-```
-
-This does not mean that `rules_python` is fetching the wheels eagerly, but it
-rather means that it is calling the PyPI server to get the Simple API response
-to get the list of all available source and wheel distributions. Once it has
-got all of the available distributions, it will select the right ones depending
-on the `sha256` values in your `requirements_lock.txt` file. The compatible
-distribution URLs will be then written to the `MODULE.bazel.lock` file. Currently
-users wishing to use the lock file with `rules_python` with this feature have
-to set an environment variable `RULES_PYTHON_OS_ARCH_LOCK_FILE=0` which will
-become default in the next release.
-
-Fetching the distribution information from the PyPI allows `rules_python` to
-know which `whl` should be used on which target platform and it will determine
-that by parsing the `whl` filename based on [PEP600], [PEP656] standards. This
-allows the user to configure the behaviour by using the following publicly
-available flags:
-* {obj}`--@rules_python//python/config_settings:py_linux_libc` for selecting the Linux libc variant.
-* {obj}`--@rules_python//python/config_settings:pip_whl` for selecting `whl` distribution preference.
-* {obj}`--@rules_python//python/config_settings:pip_whl_osx_arch` for selecting MacOS wheel preference.
-* {obj}`--@rules_python//python/config_settings:pip_whl_glibc_version` for selecting the GLIBC version compatibility.
-* {obj}`--@rules_python//python/config_settings:pip_whl_muslc_version` for selecting the musl version compatibility.
-* {obj}`--@rules_python//python/config_settings:pip_whl_osx_version` for selecting MacOS version compatibility.
-
-[bazel_downloader]: https://bazel.build/rules/lib/builtins/repository_ctx#download
-[pep600]: https://peps.python.org/pep-0600/
-[pep656]: https://peps.python.org/pep-0656/
-
 (credential-helper)=
 ### Credential Helper
 

examples/bzlmod/MODULE.bazel

@@ -104,11 +104,11 @@ register_toolchains("@uv_toolchains//:all")
 # See @rules_python//python/extensions:whl_mods.bzl attributes for more information
 # on each of the attributes.
 # You are able to set a hub name, so that you can have different modifications of the same
-# wheel in different pip hubs.
-pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip")
+# wheel in different pypi hubs.
+pypi = use_extension("@rules_python//python/extensions:pypi.bzl", "pypi")
 
 # Call whl_mods.create for the requests package.
-pip.whl_mods(
+pypi.whl_mods(
     # we are using the appended_build_content.BUILD file
     # to add content to the request wheel BUILD file.
     additive_build_content_file = "//whl_mods:appended_build_content.BUILD",
@@ -127,7 +127,7 @@ write_file(
 """
 
 # Call whl_mods.create for the wheel package.
-pip.whl_mods(
+pypi.whl_mods(
     additive_build_content = ADDITIVE_BUILD_CONTENT,
     copy_executables = {
         "@@//whl_mods:data/copy_executable.py": "copied_content/executable.py",
@@ -140,27 +140,24 @@ pip.whl_mods(
     hub_name = "whl_mods_hub",
     whl_name = "wheel",
 )
-use_repo(pip, "whl_mods_hub")
+use_repo(pypi, "whl_mods_hub")
 
-# To fetch pip dependencies, use pip.parse. We can pass in various options,
+# To fetch pypi dependencies, use pypi.install. We can pass in various options,
 # but typically we pass requirements and the Python version. The Python
 # version must have been configured by a corresponding `python.toolchain()`
 # call.
 # Alternatively, `python_interpreter_target` can be used to directly specify
 # the Python interpreter to run to resolve dependencies.
-pip.parse(
+pypi.install(
     # We can use `envsubst in the above
     envsubst = ["PIP_INDEX_URL"],
-    # Use the bazel downloader to query the simple API for downloading the sources
-    # Note, that we can use envsubst for this value.
-    experimental_index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     # One can also select a particular index for a particular package.
     # This ensures that the setup is resistant against confusion attacks.
-    # experimental_index_url_overrides = {
+    # index_url_overrides = {
     #    "my_package": "https://different-index-url.com",
     # },
     # Or you can specify extra indexes like with `pip`:
-    # experimental_extra_index_urls = [
+    # extra_index_urls = [
     #    "https://different-index-url.com",
     # ],
     experimental_requirement_cycles = {
@@ -173,15 +170,10 @@ pip.parse(
             "sphinxcontrib-serializinghtml",
         ],
     },
-    # You can use one of the values below to specify the target platform
-    # to generate the dependency graph for.
-    experimental_target_platforms = [
-        # Specifying the target platforms explicitly
-        "cp39_linux_x86_64",
-        "cp39_linux_*",
-        "cp39_*",
-    ],
     hub_name = "pip",
+    # Use the bazel downloader to query the simple API for downloading the sources
+    # Note, that we can use envsubst for this value.
+    index_url = "${PIP_INDEX_URL:-https://pypi.org/simple}",
     python_version = "3.9",
     requirements_lock = "requirements_lock_3_9.txt",
     # These modifications were created above and we
@@ -192,7 +184,7 @@ pip.parse(
         "@whl_mods_hub//:wheel.json": "wheel",
     },
 )
-pip.parse(
+pypi.install(
     experimental_requirement_cycles = {
         "sphinx": [
             "sphinx",
@@ -203,18 +195,6 @@ pip.parse(
             "sphinxcontrib-serializinghtml",
         ],
     },
-    # You can use one of the values below to specify the target platform
-    # to generate the dependency graph for.
-    experimental_target_platforms = [
-        # Using host python version
-        "linux_*",
-        "osx_*",
-        "windows_*",
-        # Or specifying an exact platform
-        "linux_x86_64",
-        # Or the following to get the `host` platform only
-        "host",
-    ],
     hub_name = "pip",
     python_version = "3.10",
     # The requirements files for each platform that we want to support.
@@ -240,7 +220,7 @@ pip.parse(
 # You can add patches that will be applied on the whl contents.
 #
 # The patches have to be in the unified-diff format.
-pip.override(
+pypi.override(
     file = "requests-2.25.1-py2.py3-none-any.whl",
     patch_strip = 1,
     patches = [
@@ -249,7 +229,7 @@ pip.override(
         "@//patches:requests_record.patch",
     ],
 )
-use_repo(pip, "pip")
+use_repo(pypi, "pip")
 
 bazel_dep(name = "other_module", version = "", repo_name = "our_other_module")
 local_path_override(