fix: use the python micro version to parse whl metadata in bzlmod

aignas · aignas · commit c86673f364b0 · 2025-04-24T09:11:26.000+09:00
Add `<micro>` version to the target platform. Instead of `cpxy_os_cpu` the target platform string format becomes `cpxy.z_os_cpu`. This is a temporary measure until we get a better API for defining target platforms. Summary: - [x] test `select_whls` function needs to be tested to ensure that the whl selection is not impacted when we have the full version in the target platform. - [ ] `download_only` legacy whl code path in `bzlmod` needs further testing. - [ ] Extra testing to ensure that the default version selection works correctly. - [x] test `whl_config_setting` handling and config setting creation. The config settings in the hub repo should not use the full version, because from the outside, the whl is compatible with all `micro` versions of a given `3.<minor_version>` of the Python interpreter. This means that the already documented config setting do not need to be changed. - [x] changelog. - [x] `pep508_deps` tests for handling the `full_python_version` correctly. Fixes #2319
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -69,6 +69,9 @@ END_UNRELEASED_TEMPLATE
   * The `sys._base_executable` value will reflect the underlying interpreter,
     not venv interpreter.
   * The {obj}`//python/runtime_env_toolchains:all` toolchain now works with it.
+* (pypi) Correctly handle `METADATA` entries when `python_full_version` is used in
+  the environment marker.
+  Fixes [#2319](https://github.com/bazel-contrib/rules_python/issues/2319).
 
 {#v0-0-0-added}
 ### Added
diff --git a/examples/bzlmod/entry_points/BUILD.bazel b/examples/bzlmod/entry_points/BUILD.bazel
@@ -1,4 +1,3 @@
-load("@python_versions//3.9:defs.bzl", py_console_script_binary_3_9 = "py_console_script_binary")
 load("@rules_python//python/entry_points:py_console_script_binary.bzl", "py_console_script_binary")
 
 # This is how you can define a `pylint` entrypoint which uses the default python version.
@@ -24,10 +23,11 @@ py_console_script_binary(
     ],
 )
 
-# A specific Python version can be forced by using the generated version-aware
-# wrappers, e.g. to force Python 3.9:
-py_console_script_binary_3_9(
+# A specific Python version can be forced by passing `python_version`
+# attribute, e.g. to force Python 3.9:
+py_console_script_binary(
     name = "yamllint",
     pkg = "@pip//yamllint:pkg",
+    python_version = "3.9",
     visibility = ["//entry_points:__subpackages__"],
 )
diff --git a/python/private/pypi/BUILD.bazel b/python/private/pypi/BUILD.bazel
@@ -103,6 +103,7 @@ bzl_library(
         "//python/private:version_label_bzl",
         "@bazel_features//:features",
         "@pythons_hub//:interpreters_bzl",
+        "@pythons_hub//:versions_bzl",
     ],
 )
 
@@ -220,7 +221,9 @@ bzl_library(
         ":pep508_evaluate_bzl",
         ":pep508_platform_bzl",
         ":pep508_requirement_bzl",
+        "//python/private:full_version_bzl",
         "//python/private:normalize_name_bzl",
+        "@pythons_hub//:versions_bzl",
     ],
 )
 
diff --git a/python/private/pypi/config_settings.bzl b/python/private/pypi/config_settings.bzl
@@ -42,6 +42,8 @@ specialized is as follows:
 * `:is_cp3<minor_version>_abi3_<platform_suffix>`
 * `:is_cp3<minor_version>_cp3<minor_version>_<platform_suffix>` and `:is_cp3<minor_version>_cp3<minor_version>t_<platform_suffix>`
 
+Optionally instead of `<minor_version>` there sometimes may be `<minor_version>.<micro_version>` used in order to fully specify the versions
+
 The specialization of free-threaded vs non-free-threaded wheels is the same as
 they are just variants of each other. The same goes for the specialization of
 `musllinux` vs `manylinux`.
diff --git a/python/private/pypi/extension.bzl b/python/private/pypi/extension.bzl
@@ -16,7 +16,9 @@
 
 load("@bazel_features//:features.bzl", "bazel_features")
 load("@pythons_hub//:interpreters.bzl", "INTERPRETER_LABELS")
+load("@pythons_hub//:versions.bzl", "MINOR_MAPPING")
 load("//python/private:auth.bzl", "AUTH_ATTRS")
+load("//python/private:full_version.bzl", "full_version")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load("//python/private:repo_utils.bzl", "repo_utils")
 load("//python/private:semver.bzl", "semver")
@@ -68,6 +70,7 @@ def _create_whl_repos(
         pip_attr,
         whl_overrides,
         available_interpreters = INTERPRETER_LABELS,
+        minor_mapping = MINOR_MAPPING,
         get_index_urls = None):
     """create all of the whl repositories
 
@@ -159,8 +162,10 @@ def _create_whl_repos(
             requirements_osx = pip_attr.requirements_darwin,
             requirements_windows = pip_attr.requirements_windows,
             extra_pip_args = pip_attr.extra_pip_args,
-            # TODO @aignas 2025-04-15: pass the full version into here
-            python_version = major_minor,
+            python_version = full_version(
+                version = pip_attr.python_version,
+                minor_mapping = minor_mapping,
+            ),
             logger = logger,
         ),
         extra_pip_args = pip_attr.extra_pip_args,
@@ -304,9 +309,6 @@ def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patt
     if requirement.extra_pip_args:
         args["extra_pip_args"] = requirement.extra_pip_args
 
-    if download_only:
-        args.setdefault("experimental_target_platforms", requirement.target_platforms)
-
     target_platforms = requirement.target_platforms if multiple_requirements_for_whl else []
     repo_name = pypi_repo_name(
         normalize_name(requirement.distribution),
diff --git a/python/private/pypi/pep508_deps.bzl b/python/private/pypi/pep508_deps.bzl
@@ -15,14 +15,23 @@
 """This module is for implementing PEP508 compliant METADATA deps parsing.
 """
 
-load("@pythons_hub//:versions.bzl", "DEFAULT_PYTHON_VERSION")
+load("@pythons_hub//:versions.bzl", "DEFAULT_PYTHON_VERSION", "MINOR_MAPPING")
+load("//python/private:full_version.bzl", "full_version")
 load("//python/private:normalize_name.bzl", "normalize_name")
 load(":pep508_env.bzl", "env")
 load(":pep508_evaluate.bzl", "evaluate")
 load(":pep508_platform.bzl", "platform", "platform_from_str")
 load(":pep508_requirement.bzl", "requirement")
 
-def deps(name, *, requires_dist, platforms = [], extras = [], excludes = [], default_python_version = None):
+def deps(
+        name,
+        *,
+        requires_dist,
+        platforms = [],
+        extras = [],
+        excludes = [],
+        default_python_version = None,
+        minor_mapping = MINOR_MAPPING):
     """Parse the RequiresDist from wheel METADATA
 
     Args:
@@ -33,6 +42,9 @@ def deps(name, *, requires_dist, platforms = [], extras = [], excludes = [], def
         extras: {type}`list[str]` the requested extras to generate targets for.
         platforms: {type}`list[str]` the list of target platform strings.
         default_python_version: {type}`str` the host python version.
+        minor_mapping: {type}`type[str, str]` the minor mapping to use when
+            resolving to the full python version as DEFAULT_PYTHON_VERSION can by
+            of format `3.x`.
 
     Returns:
         A struct with attributes:
@@ -52,17 +64,22 @@ def deps(name, *, requires_dist, platforms = [], extras = [], excludes = [], def
     # drop self edges
     excludes = [name] + [normalize_name(x) for x in excludes]
 
-    default_python_version = default_python_version or DEFAULT_PYTHON_VERSION
+    default_python_version = default_python_version or default_python_version
+    if default_python_version:
+        # if it is not bzlmod, then DEFAULT_PYTHON_VERSION may be unset
+        default_python_version = full_version(
+            version = default_python_version or DEFAULT_PYTHON_VERSION,
+            minor_mapping = minor_mapping,
+        )
     platforms = [
         platform_from_str(p, python_version = default_python_version)
         for p in platforms
     ]
 
     abis = sorted({p.abi: True for p in platforms if p.abi})
     if default_python_version and len(abis) > 1:
-        _, _, minor_version = default_python_version.partition(".")
-        minor_version, _, _ = minor_version.partition(".")
-        default_abi = "cp3" + minor_version
+        _, _, tail = default_python_version.partition(".")
+        default_abi = "cp3" + tail
     elif len(abis) > 1:
         fail(
             "all python versions need to be specified explicitly, got: {}".format(platforms),
diff --git a/python/private/pypi/pkg_aliases.bzl b/python/private/pypi/pkg_aliases.bzl
@@ -371,6 +371,9 @@ def get_filename_config_settings(
 
         abi = parsed.abi_tag
 
+        # TODO @aignas 2025-04-20: test
+        abi, _, _ = abi.partition(".")
+
         if parsed.platform_tag == "any":
             prefixes = ["{}{}_any".format(py, abi)]
         else:
diff --git a/python/private/pypi/render_pkg_aliases.bzl b/python/private/pypi/render_pkg_aliases.bzl
@@ -143,6 +143,18 @@ def render_pkg_aliases(*, aliases, requirement_cycles = None, extra_hub_aliases
         files["_groups/BUILD.bazel"] = generate_group_library_build_bazel("", requirement_cycles)
     return files
 
+def _major_minor(python_version):
+    major, _, tail = python_version.partition(".")
+    minor, _, _ = tail.partition(".")
+    return "{}.{}".format(major, minor)
+
+def _major_minor_versions(python_versions):
+    if not python_versions:
+        return []
+
+    # Use a dict as a simple set
+    return sorted({_major_minor(v): None for v in python_versions})
+
 def render_multiplatform_pkg_aliases(*, aliases, **kwargs):
     """Render the multi-platform pkg aliases.
 
@@ -174,7 +186,7 @@ def render_multiplatform_pkg_aliases(*, aliases, **kwargs):
         glibc_versions = flag_versions.get("glibc_versions", []),
         muslc_versions = flag_versions.get("muslc_versions", []),
         osx_versions = flag_versions.get("osx_versions", []),
-        python_versions = flag_versions.get("python_versions", []),
+        python_versions = _major_minor_versions(flag_versions.get("python_versions", [])),
         target_platforms = flag_versions.get("target_platforms", []),
         visibility = ["//:__subpackages__"],
     )
diff --git a/python/private/pypi/requirements_files_by_platform.bzl b/python/private/pypi/requirements_files_by_platform.bzl
@@ -91,13 +91,12 @@ def _platforms_from_args(extra_pip_args):
     return list(platforms.keys())
 
 def _platform(platform_string, python_version = None):
-    if not python_version or platform_string.startswith("cp3"):
+    if not python_version or platform_string.startswith("cp"):
         return platform_string
 
-    _, _, tail = python_version.partition(".")
-    minor, _, _ = tail.partition(".")
+    major, _, tail = python_version.partition(".")
 
-    return "cp3{}_{}".format(minor, platform_string)
+    return "cp{}{}_{}".format(major, tail, platform_string)
 
 def requirements_files_by_platform(
         *,
diff --git a/python/private/pypi/whl_config_setting.bzl b/python/private/pypi/whl_config_setting.bzl
@@ -35,10 +35,20 @@ def whl_config_setting(*, version = None, config_setting = None, filename = None
         a struct with the validated and parsed values.
     """
     if target_platforms:
-        for p in target_platforms:
+        target_platforms_input = target_platforms
+        target_platforms = []
+        for p in target_platforms_input:
             if not p.startswith("cp"):
                 fail("target_platform should start with 'cp' denoting the python version, got: " + p)
 
+            abi, _, tail = p.partition("_")
+
+            # drop the micro version here, currently there is no usecase to use
+            # multiple python interpreters with the same minor version but
+            # different micro version.
+            abi, _, _ = abi.partition(".")
+            target_platforms.append("{}_{}".format(abi, tail))
+
     return struct(
         config_setting = config_setting,
         filename = filename,
diff --git a/python/private/pypi/whl_library_targets.bzl b/python/private/pypi/whl_library_targets.bzl
@@ -369,26 +369,22 @@ def _config_settings(dependencies_by_platform, native = native, **kwargs):
         if p.startswith("@") or p.endswith("default"):
             continue
 
+        # TODO @aignas 2025-04-20: add tests here
         abi, _, tail = p.partition("_")
         if not abi.startswith("cp"):
             tail = p
             abi = ""
-
         os, _, arch = tail.partition("_")
-        os = "" if os == "anyos" else os
-        arch = "" if arch == "anyarch" else arch
 
         _kwargs = dict(kwargs)
-        if arch:
-            _kwargs.setdefault("constraint_values", []).append("@platforms//cpu:{}".format(arch))
-        if os:
-            _kwargs.setdefault("constraint_values", []).append("@platforms//os:{}".format(os))
+        _kwargs["constraint_values"] = [
+            "@platforms//cpu:{}".format(arch),
+            "@platforms//os:{}".format(os),
+        ]
 
         if abi:
             _kwargs["flag_values"] = {
-                "@rules_python//python/config_settings:python_version_major_minor": "3.{minor_version}".format(
-                    minor_version = abi[len("cp3"):],
-                ),
+                str(Label("//python/config_settings:python_version")): "3.{}".format(abi[len("cp3"):])
             }
 
         native.config_setting(
diff --git a/python/private/pypi/whl_target_platforms.bzl b/python/private/pypi/whl_target_platforms.bzl
@@ -75,8 +75,11 @@ def select_whls(*, whls, want_platforms = [], logger = None):
             fail("expected all platforms to start with ABI, but got: {}".format(p))
 
         abi, _, os_cpu = p.partition("_")
+        abi, _, _ = abi.partition(".")
         _want_platforms[os_cpu] = None
-        _want_platforms[p] = None
+
+        # TODO @aignas 2025-04-20: add a test
+        _want_platforms["{}_{}".format(abi, os_cpu)] = None
 
         version_limit_candidate = int(abi[3:])
         if not version_limit:
diff --git a/tests/pypi/extension/extension_tests.bzl b/tests/pypi/extension/extension_tests.bzl
@@ -157,6 +157,7 @@ def _test_simple(env):
         available_interpreters = {
             "python_3_15_host": "unit_test_interpreter_target",
         },
+        minor_mapping = {"3.15": "3.15.19"},
     )
 
     pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
@@ -204,6 +205,7 @@ def _test_simple_multiple_requirements(env):
         available_interpreters = {
             "python_3_15_host": "unit_test_interpreter_target",
         },
+        minor_mapping = {"3.15": "3.15.19"},
     )
 
     pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
@@ -270,6 +272,7 @@ torch==2.4.1 ; platform_machine != 'x86_64' \
         available_interpreters = {
             "python_3_15_host": "unit_test_interpreter_target",
         },
+        minor_mapping = {"3.15": "3.15.19"},
     )
 
     pypi.exposed_packages().contains_exactly({"pypi": ["torch"]})
@@ -392,6 +395,7 @@ torch==2.4.1+cpu ; platform_machine == 'x86_64' \
         available_interpreters = {
             "python_3_12_host": "unit_test_interpreter_target",
         },
+        minor_mapping = {"3.12": "3.12.19"},
         simpleapi_download = mocksimpleapi_download,
     )
 
@@ -515,6 +519,7 @@ simple==0.0.3 \
         available_interpreters = {
             "python_3_15_host": "unit_test_interpreter_target",
         },
+        minor_mapping = {"3.15": "3.15.19"},
     )
 
     pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
@@ -544,23 +549,22 @@ simple==0.0.3 \
         "pypi_315_extra": {
             "dep_template": "@pypi//{name}:{target}",
             "download_only": True,
-            "experimental_target_platforms": ["cp315_linux_x86_64"],
+            # TODO @aignas 2025-04-20: ensure that this is in the hub repo
+            # "experimental_target_platforms": ["cp315_linux_x86_64"],
             "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"],
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "extra==0.0.1 --hash=sha256:deadb00f",
         },
         "pypi_315_simple_linux_x86_64": {
             "dep_template": "@pypi//{name}:{target}",
             "download_only": True,
-            "experimental_target_platforms": ["cp315_linux_x86_64"],
             "extra_pip_args": ["--platform=manylinux_2_17_x86_64", "--python-version=315", "--implementation=cp", "--abi=cp315"],
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "simple==0.0.1 --hash=sha256:deadbeef",
         },
         "pypi_315_simple_osx_aarch64": {
             "dep_template": "@pypi//{name}:{target}",
             "download_only": True,
-            "experimental_target_platforms": ["cp315_osx_aarch64"],
             "extra_pip_args": ["--platform=macosx_10_9_arm64", "--python-version=315", "--implementation=cp", "--abi=cp315"],
             "python_interpreter_target": "unit_test_interpreter_target",
             "requirement": "simple==0.0.3 --hash=sha256:deadbaaf",
@@ -648,6 +652,7 @@ git_dep @ git+https://git.server/repo/project@deadbeefdeadbeef
         available_interpreters = {
             "python_3_15_host": "unit_test_interpreter_target",
         },
+        minor_mapping = {"3.15": "3.15.19"},
         simpleapi_download = mocksimpleapi_download,
     )
 
@@ -850,6 +855,7 @@ optimum[onnxruntime-gpu]==1.17.1 ; sys_platform == 'linux'
         available_interpreters = {
             "python_3_15_host": "unit_test_interpreter_target",
         },
+        minor_mapping = {"3.15": "3.15.19"},
     )
 
     pypi.exposed_packages().contains_exactly({"pypi": []})
diff --git a/tests/pypi/pep508/deps_tests.bzl b/tests/pypi/pep508/deps_tests.bzl
diff --git a/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl b/tests/pypi/render_pkg_aliases/render_pkg_aliases_test.bzl
diff --git a/tests/pypi/whl_target_platforms/select_whl_tests.bzl b/tests/pypi/whl_target_platforms/select_whl_tests.bzl
