Skip to content

Commit a916199

Browse files
aignasaignas
committed
fix: prefer X.Y over X.Y.Z versions in python.toolchain
Before this PR the `bzlmod` and legacy setups would only expose the multi-version python toolchain aliases as the string that is passed to the `python_register_multi_toolchains` function. This meant that if the user decided to pass the full version (e.g. `3.11.1`) then they had to import the version aware `py_*` defs via `@foo//3.11.1:defs.bzl`. This PR changes it such that the user can still do the old way of importing but we print a deprecation warning and ask the user to use `3.11:defs.bzl` instead which better reflects how the multi-version toolchain should be used. This also means that the PRs bumping the minor versions like in #1352 won't need updating Python code elsewhere. Whilst at it, we introduce a validation that disallows multiple Python toolchains of the same `X.Y` version. This is handled in the `bzlmod` by printing warnings that the toolchains are ignored, whilst the non-bzlmod users will get validation errors, which is a potentially breaking change. Fixes #1339
1 parent fabb65f commit a916199

File tree

9 files changed

+125
-34
lines changed

9 files changed

+125
-34
lines changed

‎WORKSPACE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -72,7 +72,7 @@ _py_gazelle_deps()
7272
# Install twine for our own runfiles wheel publishing.
7373
# Eventually we might want to install twine automatically for users too, see:
7474
# https://github.com/bazelbuild/rules_python/issues/1016.
75-
load("@python//3.11.1:defs.bzl", "interpreter")
75+
load("@python//3.11:defs.bzl", "interpreter")
7676
load("@rules_python//python:pip.bzl", "pip_parse")
7777

7878
pip_parse(

‎examples/bzlmod/MODULE.bazel

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,8 @@ python.toolchain(
2828
# work in progress.
2929
python.toolchain(
3030
configure_coverage_tool = True,
31-
python_version = "3.10",
31+
# One can also provide the full Python version.
32+
python_version = "3.10.9",
3233
)
3334

3435
# You only need to load this repositories if you are using multiple Python versions.

‎examples/bzlmod/other_module/MODULE.bazel

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ PYTHON_NAME_311 = "python_3_11"
2727
python = use_extension("@rules_python//python/extensions:python.bzl", "python")
2828
python.toolchain(
2929
configure_coverage_tool = True,
30-
python_version = "3.9",
30+
python_version = "3.9.10",
3131
)
3232
python.toolchain(
3333
configure_coverage_tool = True,

‎examples/multi_python_versions/WORKSPACE

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ python_register_multi_toolchains(
2222
"3.8",
2323
"3.9",
2424
"3.10",
25-
"3.11",
25+
"3.11.1",
2626
],
2727
register_coverage_tool = True,
2828
)

‎python/extensions/private/pythons_hub.bzl

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,8 @@
1414

1515
"Repo rule used by bzlmod extension to create a repo that has a map of Python interpreters and their labels"
1616

17-
load("//python:versions.bzl", "MINOR_MAPPING", "WINDOWS_NAME")
17+
load("//python:versions.bzl", "WINDOWS_NAME")
18+
load("//python/private:full_version.bzl", "full_version")
1819
load(
1920
"//python/private:toolchains_repo.bzl",
2021
"get_host_os_arch",
@@ -28,12 +29,6 @@ def _have_same_length(*lists):
2829
fail("expected at least one list")
2930
return len({len(length): None for length in lists}) == 1
3031

31-
def _get_version(python_version):
32-
# we need to get the MINOR_MAPPING or use the full version
33-
if python_version in MINOR_MAPPING:
34-
python_version = MINOR_MAPPING[python_version]
35-
return python_version
36-
3732
def _python_toolchain_build_file_content(
3833
prefixes,
3934
python_versions,
@@ -55,7 +50,7 @@ def _python_toolchain_build_file_content(
5550
# build the toolchain content by calling python_toolchain_build_file_content
5651
return "\n".join([python_toolchain_build_file_content(
5752
prefix = prefixes[i],
58-
python_version = _get_version(python_versions[i]),
53+
python_version = full_version(python_versions[i]),
5954
set_python_version_constraint = set_python_version_constraints[i],
6055
user_repository_name = user_repository_names[i],
6156
rules_python = rules_python,

‎python/extensions/python.bzl

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@
1616

1717
load("//python:repositories.bzl", "python_register_toolchains")
1818
load("//python/extensions/private:pythons_hub.bzl", "hub_repo")
19+
load("//python/private:full_version.bzl", "full_version")
1920
load("//python/private:toolchains_repo.bzl", "multi_toolchain_aliases")
21+
load("//python/private:version_label.bzl", "version_label")
2022

2123
# This limit can be increased essentially arbitrarily, but doing so will cause a rebuild of all
2224
# targets using any of these toolchains due to the changed repository name.
@@ -74,28 +76,30 @@ def _python_impl(module_ctx):
7476
module_toolchain_versions = []
7577

7678
for toolchain_attr in mod.tags.toolchain:
77-
toolchain_version = toolchain_attr.python_version
78-
toolchain_name = "python_" + toolchain_version.replace(".", "_")
79+
toolchain_version = full_version(toolchain_attr.python_version)
80+
toolchain_version_short, _, _ = toolchain_version.rpartition(".")
81+
toolchain_name = "python_" + version_label(toolchain_version, sep = "_")
7982

8083
# Duplicate versions within a module indicate a misconfigured module.
81-
if toolchain_version in module_toolchain_versions:
82-
_fail_duplicate_module_toolchain_version(toolchain_version, mod.name)
83-
module_toolchain_versions.append(toolchain_version)
84+
if toolchain_version_short in module_toolchain_versions:
85+
_fail_duplicate_module_toolchain_version(toolchain_version_short, mod.name)
86+
module_toolchain_versions.append(toolchain_version_short)
8487

8588
# Ignore version collisions in the global scope because there isn't
8689
# much else that can be done. Modules don't know and can't control
8790
# what other modules do, so the first in the dependency graph wins.
88-
if toolchain_version in global_toolchain_versions:
91+
if toolchain_version_short in global_toolchain_versions:
8992
_warn_duplicate_global_toolchain_version(
90-
toolchain_version,
91-
first = global_toolchain_versions[toolchain_version],
93+
first = global_toolchain_versions[toolchain_version_short],
9294
second_toolchain_name = toolchain_name,
9395
second_module_name = mod.name,
96+
second_toolchain_version = toolchain_version,
9497
)
9598
continue
96-
global_toolchain_versions[toolchain_version] = struct(
99+
global_toolchain_versions[toolchain_version_short] = struct(
97100
toolchain_name = toolchain_name,
98101
module_name = mod.name,
102+
version = toolchain_version,
99103
)
100104

101105
# Only the root module and rules_python are allowed to specify the default
@@ -177,17 +181,18 @@ def _fail_duplicate_module_toolchain_version(version, module):
177181
module = module,
178182
))
179183

180-
def _warn_duplicate_global_toolchain_version(version, first, second_toolchain_name, second_module_name):
184+
def _warn_duplicate_global_toolchain_version(first, second_toolchain_name, second_module_name, second_toolchain_version):
181185
_print_warn((
182-
"Ignoring toolchain '{second_toolchain}' from module '{second_module}': " +
186+
"Ignoring toolchain '{second_toolchain}' ({second_version}) from module '{second_module}': " +
183187
"Toolchain '{first_toolchain}' from module '{first_module}' " +
184-
"already registered Python version {version} and has precedence"
188+
"already registered Python version {first_version} and has precedence"
185189
).format(
186190
first_toolchain = first.toolchain_name,
187191
first_module = first.module_name,
188192
second_module = second_module_name,
189193
second_toolchain = second_toolchain_name,
190-
version = version,
194+
first_version = first.version,
195+
second_version = second_toolchain_version,
191196
))
192197

193198
def _fail_multiple_default_toolchains(first, second):

‎python/pip.bzl

Lines changed: 14 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,9 @@ load("//python/pip_install:pip_repository.bzl", "pip_repository", _package_annot
1717
load("//python/pip_install:repositories.bzl", "pip_install_dependencies")
1818
load("//python/pip_install:requirements.bzl", _compile_pip_requirements = "compile_pip_requirements")
1919
load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")
20+
load("//python/private:full_version.bzl", "full_version")
2021
load("//python/private:render_pkg_aliases.bzl", "NO_MATCH_ERROR_MESSAGE_TEMPLATE")
21-
load(":versions.bzl", "MINOR_MAPPING")
22+
load("//python/private:version_label.bzl", "version_label")
2223

2324
compile_pip_requirements = _compile_pip_requirements
2425
package_annotation = _package_annotation
@@ -166,7 +167,7 @@ def _multi_pip_parse_impl(rctx):
166167
install_deps_calls = []
167168
process_requirements_calls = []
168169
for python_version, pypi_repository in rctx.attr.pip_parses.items():
169-
sanitized_python_version = python_version.replace(".", "_")
170+
sanitized_python_version = version_label(python_version, sep = "_")
170171
load_statement = """\
171172
load(
172173
"@{pypi_repository}//:requirements.bzl",
@@ -184,7 +185,7 @@ _process_requirements(
184185
repo_prefix = "{pypi_repository}_",
185186
)""".format(
186187
pypi_repository = pypi_repository,
187-
python_version = python_version,
188+
python_version = version_label(python_version, sep = "."),
188189
sanitized_python_version = sanitized_python_version,
189190
)
190191
process_requirements_calls.append(process_requirements_call)
@@ -295,7 +296,7 @@ alias(
295296
for [python_version, repo_prefix] in version_map:
296297
alias.append("""\
297298
"@{rules_python}//python/config_settings:is_python_{full_python_version}": "{actual}",""".format(
298-
full_python_version = MINOR_MAPPING[python_version] if python_version in MINOR_MAPPING else python_version,
299+
full_python_version = full_version(python_version),
299300
actual = "@{repo_prefix}{wheel_name}//:{alias_name}".format(
300301
repo_prefix = repo_prefix,
301302
wheel_name = wheel_name,
@@ -363,13 +364,21 @@ def multi_pip_parse(name, default_version, python_versions, python_interpreter_t
363364
The internal implementation of multi_pip_parse repository rule.
364365
"""
365366
pip_parses = {}
367+
python_interpreter_target = {
368+
full_version(v): t
369+
for v, t in python_interpreter_target.items()
370+
}
371+
requirements_lock = {
372+
full_version(v): t
373+
for v, t in requirements_lock.items()
374+
}
366375
for python_version in python_versions:
367376
if not python_version in python_interpreter_target:
368377
fail("Missing python_interpreter_target for Python version %s in '%s'" % (python_version, name))
369378
if not python_version in requirements_lock:
370379
fail("Missing requirements_lock for Python version %s in '%s'" % (python_version, name))
371380

372-
pip_parse_name = name + "_" + python_version.replace(".", "_")
381+
pip_parse_name = name + "_" + version_label(python_version, sep = "_")
373382
pip_parse(
374383
name = pip_parse_name,
375384
python_interpreter_target = python_interpreter_target[python_version],

‎python/private/full_version.bzl

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2022 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"A small helper to ensure that we are working with full versions."
16+
17+
load("//python:versions.bzl", "MINOR_MAPPING")
18+
19+
def full_version(version):
20+
"""Return a full version.
21+
22+
Args:
23+
version: the version in `X.Y` or `X.Y.Z` format.
24+
25+
Returns:
26+
a full version given the version string. If the string is already a
27+
major version then we return it as is.
28+
"""
29+
parts = version.split(".")
30+
if len(parts) == 2:
31+
return MINOR_MAPPING[version]
32+
elif len(parts) == 3:
33+
return version
34+
else:
35+
fail("Unknown version format: {}".format(version))

‎python/private/toolchains_repo.bzl

Lines changed: 50 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@ load(
3030
"PLATFORMS",
3131
"WINDOWS_NAME",
3232
)
33+
load("//python/private:full_version.bzl", "full_version")
34+
load("//python/private:version_label.bzl", "version_label")
3335

3436
def get_repository_name(repository_workspace):
3537
dummy_label = "//:_"
@@ -232,12 +234,48 @@ toolchain_aliases = repository_rule(
232234
},
233235
)
234236

237+
def _validate_uniqueness(python_versions):
238+
"""Validate that the Python versions have unique `X.Y` entries
239+
240+
Otherwise the repository creation will fail with weird "file-exists" errors
241+
and this allows us to have an actionable error message to the user.
242+
243+
Args:
244+
python_versions: The list of python versions
245+
"""
246+
minor_versions = {}
247+
for version in python_versions:
248+
short_version = version_label(version, sep = ".")
249+
versions = minor_versions.get(short_version, [])
250+
versions.append(version)
251+
minor_versions[short_version] = versions
252+
253+
non_unique_versions = {
254+
k: v
255+
for k, v in minor_versions.items()
256+
if len(v) != 1
257+
}
258+
259+
if non_unique_versions:
260+
fail(
261+
"Expected all python versions map to unique 'X.Y' values but found duplicates:\n" +
262+
"".join([
263+
" '{}': {}\n".format(
264+
minor_version,
265+
", ".join(versions),
266+
)
267+
for minor_version, versions in non_unique_versions.items()
268+
]) +
269+
"Please remove duplicates.",
270+
)
271+
235272
def _multi_toolchain_aliases_impl(rctx):
236273
rules_python = rctx.attr._rules_python_workspace.workspace_name
237274

275+
_validate_uniqueness(rctx.attr.python_versions.keys())
276+
238277
for python_version, repository_name in rctx.attr.python_versions.items():
239-
file = "{}/defs.bzl".format(python_version)
240-
rctx.file(file, content = """\
278+
content = """\
241279
# Generated by python/private/toolchains_repo.bzl
242280
243281
load(
@@ -256,9 +294,17 @@ py_binary = _py_binary
256294
py_test = _py_test
257295
""".format(
258296
repository_name = repository_name,
259-
))
297+
)
298+
299+
short_version = version_label(python_version, sep = ".")
300+
warning = "\nprint(\"DEPRECATED: please migrate your code to use '@{}//{}:defs.bzl'\")".format(rctx.attr.name, short_version)
301+
302+
rctx.file("{}/defs.bzl".format(python_version), content = content + warning)
260303
rctx.file("{}/BUILD.bazel".format(python_version), "")
261304

305+
rctx.file("{}/defs.bzl".format(short_version), content = content)
306+
rctx.file("{}/BUILD.bazel".format(short_version), "")
307+
262308
pip_bzl = """\
263309
# Generated by python/private/toolchains_repo.bzl
264310
@@ -273,7 +319,7 @@ def multi_pip_parse(name, requirements_lock, **kwargs):
273319
)
274320
275321
""".format(
276-
python_versions = rctx.attr.python_versions.keys(),
322+
python_versions = [full_version(v) for v in rctx.attr.python_versions.keys()],
277323
rules_python = rules_python,
278324
)
279325
rctx.file("pip.bzl", content = pip_bzl)

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy