Skip to content

Commit 67e233f

Browse files
aignasaignas
authored
fix(pypi): output only necessary target_platforms (bazel-contrib#2710)
This change reduces the number of lines we are going to write to the MODULE.bazel.lock file by not writing `experimental_target_platforms` to the lock file that eventually get discarded in the `whl_library` if the wheel is platform specific [1]. This means that the tests will become more easy to understand, but technically this is a no-op change, only resulting in a smaller lock file: ``` $ wc -l MODULE.bazel.lock 6536 MODULE.bazel.lock $ bazel mod deps --lockfile_mode=refresh ... $ wc -l MODULE.bazel.lock 6154 MODULE.bazel.lock ``` Work related to bazel-contrib#2622 [1]: https://github.com/bazel-contrib/rules_python/blob/09145b9f628d482246eaa70421bf0cbae9acb096/python/private/pypi/whl_library.bzl#L337
1 parent 5d6827e commit 67e233f

File tree

4 files changed

+190
-3
lines changed

4 files changed

+190
-3
lines changed

‎CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,8 @@ Unreleased changes template.
6464
* 3.12.9
6565
* 3.13.2
6666
* (pypi) Use `xcrun xcodebuild --showsdks` to find XCode root.
67+
* (pypi) The `bzlmod` extension will now generate smaller lock files for when
68+
using `experimental_index_url`.
6769
* (toolchains) Remove all but `3.8.20` versions of the Python `3.8` interpreter who has
6870
reached EOL. If users still need other versions of the `3.8` interpreter, please supply
6971
the URLs manually {bzl:ob}`python.toolchain` or {bzl:obj}`python_register_toolchains` calls.

‎python/private/pypi/BUILD.bazel

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ bzl_library(
9393
":whl_config_setting_bzl",
9494
":whl_library_bzl",
9595
":whl_repo_name_bzl",
96+
":whl_target_platforms_bzl",
9697
"//python/private:full_version_bzl",
9798
"//python/private:normalize_name_bzl",
9899
"//python/private:semver_bzl",

‎python/private/pypi/extension.bzl

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ load(":simpleapi_download.bzl", "simpleapi_download")
3232
load(":whl_config_setting.bzl", "whl_config_setting")
3333
load(":whl_library.bzl", "whl_library")
3434
load(":whl_repo_name.bzl", "pypi_repo_name", "whl_repo_name")
35+
load(":whl_target_platforms.bzl", "whl_target_platforms")
3536

3637
def _major_minor_version(version):
3738
version = semver(version)
@@ -296,9 +297,18 @@ def _whl_repos(*, requirement, whl_library_args, download_only, netrc, auth_patt
296297

297298
# Pure python wheels or sdists may need to have a platform here
298299
target_platforms = None
299-
if distribution.filename.endswith("-any.whl") or not distribution.filename.endswith(".whl"):
300-
if multiple_requirements_for_whl:
301-
target_platforms = requirement.target_platforms
300+
if distribution.filename.endswith(".whl") and not distribution.filename.endswith("-any.whl"):
301+
parsed_whl = parse_whl_name(distribution.filename)
302+
whl_platforms = whl_target_platforms(
303+
platform_tag = parsed_whl.platform_tag,
304+
)
305+
args["experimental_target_platforms"] = [
306+
p
307+
for p in requirement.target_platforms
308+
if [None for wp in whl_platforms if p.endswith(wp.target_platform)]
309+
]
310+
elif multiple_requirements_for_whl:
311+
target_platforms = requirement.target_platforms
302312

303313
repo_name = whl_repo_name(
304314
distribution.filename,

‎tests/pypi/extension/extension_tests.bzl

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
load("@rules_testing//lib:test_suite.bzl", "test_suite")
1818
load("@rules_testing//lib:truth.bzl", "subjects")
1919
load("//python/private/pypi:extension.bzl", "parse_modules") # buildifier: disable=bzl-visibility
20+
load("//python/private/pypi:parse_simpleapi_html.bzl", "parse_simpleapi_html") # buildifier: disable=bzl-visibility
2021
load("//python/private/pypi:whl_config_setting.bzl", "whl_config_setting") # buildifier: disable=bzl-visibility
2122

2223
_tests = []
@@ -332,6 +333,179 @@ torch==2.4.1 ; platform_machine != 'x86_64' \
332333

333334
_tests.append(_test_simple_with_markers)
334335

336+
def _test_torch_experimental_index_url(env):
337+
def mocksimpleapi_download(*_, **__):
338+
return {
339+
"torch": parse_simpleapi_html(
340+
url = "https://torch.index",
341+
content = """\
342+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp310-cp310-linux_x86_64.whl#sha256=833490a28ac156762ed6adaa7c695879564fa2fd0dc51bcf3fdb2c7b47dc55e6">torch-2.4.1+cpu-cp310-cp310-linux_x86_64.whl</a><br/>
343+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp310-cp310-win_amd64.whl#sha256=1dd062d296fb78aa7cfab8690bf03704995a821b5ef69cfc807af5c0831b4202">torch-2.4.1+cpu-cp310-cp310-win_amd64.whl</a><br/>
344+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp311-cp311-linux_x86_64.whl#sha256=2b03e20f37557d211d14e3fb3f71709325336402db132a1e0dd8b47392185baf">torch-2.4.1+cpu-cp311-cp311-linux_x86_64.whl</a><br/>
345+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp311-cp311-win_amd64.whl#sha256=76a6fe7b10491b650c630bc9ae328df40f79a948296b41d3b087b29a8a63cbad">torch-2.4.1+cpu-cp311-cp311-win_amd64.whl</a><br/>
346+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl#sha256=8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364">torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl</a><br/>
347+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl#sha256=3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97">torch-2.4.1+cpu-cp312-cp312-win_amd64.whl</a><br/>
348+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp38-cp38-linux_x86_64.whl#sha256=0c0a7cc4f7c74ff024d5a5e21230a01289b65346b27a626f6c815d94b4b8c955">torch-2.4.1+cpu-cp38-cp38-linux_x86_64.whl</a><br/>
349+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp38-cp38-win_amd64.whl#sha256=330e780f478707478f797fdc82c2a96e9b8c5f60b6f1f57bb6ad1dd5b1e7e97e">torch-2.4.1+cpu-cp38-cp38-win_amd64.whl</a><br/>
350+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp39-cp39-linux_x86_64.whl#sha256=3c99506980a2fb4b634008ccb758f42dd82f93ae2830c1e41f64536e310bf562">torch-2.4.1+cpu-cp39-cp39-linux_x86_64.whl</a><br/>
351+
<a href="/whl/cpu/torch-2.4.1%2Bcpu-cp39-cp39-win_amd64.whl#sha256=c4f2c3c026e876d4dad7629170ec14fff48c076d6c2ae0e354ab3fdc09024f00">torch-2.4.1+cpu-cp39-cp39-win_amd64.whl</a><br/>
352+
<a href="/whl/cpu/torch-2.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=fa27b048d32198cda6e9cff0bf768e8683d98743903b7e5d2b1f5098ded1d343">torch-2.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
353+
<a href="/whl/cpu/torch-2.4.1-cp310-none-macosx_11_0_arm64.whl#sha256=d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd">torch-2.4.1-cp310-none-macosx_11_0_arm64.whl</a><br/>
354+
<a href="/whl/cpu/torch-2.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=30be2844d0c939161a11073bfbaf645f1c7cb43f62f46cc6e4df1c119fb2a798">torch-2.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
355+
<a href="/whl/cpu/torch-2.4.1-cp311-none-macosx_11_0_arm64.whl#sha256=ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea">torch-2.4.1-cp311-none-macosx_11_0_arm64.whl</a><br/>
356+
<a href="/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a">torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
357+
<a href="/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl#sha256=72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d">torch-2.4.1-cp312-none-macosx_11_0_arm64.whl</a><br/>
358+
<a href="/whl/cpu/torch-2.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=56ad2a760b7a7882725a1eebf5657abbb3b5144eb26bcb47b52059357463c548">torch-2.4.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
359+
<a href="/whl/cpu/torch-2.4.1-cp38-none-macosx_11_0_arm64.whl#sha256=5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71">torch-2.4.1-cp38-none-macosx_11_0_arm64.whl</a><br/>
360+
<a href="/whl/cpu/torch-2.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl#sha256=1495132f30f722af1a091950088baea383fe39903db06b20e6936fd99402803e">torch-2.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl</a><br/>
361+
<a href="/whl/cpu/torch-2.4.1-cp39-none-macosx_11_0_arm64.whl#sha256=a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec">torch-2.4.1-cp39-none-macosx_11_0_arm64.whl</a><br/>
362+
""",
363+
),
364+
}
365+
366+
pypi = _parse_modules(
367+
env,
368+
module_ctx = _mock_mctx(
369+
_mod(
370+
name = "rules_python",
371+
parse = [
372+
_parse(
373+
hub_name = "pypi",
374+
python_version = "3.12",
375+
experimental_index_url = "https://torch.index",
376+
requirements_lock = "universal.txt",
377+
),
378+
],
379+
),
380+
read = lambda x: {
381+
"universal.txt": """\
382+
torch==2.4.1 ; platform_machine != 'x86_64' \
383+
--hash=sha256:1495132f30f722af1a091950088baea383fe39903db06b20e6936fd99402803e \
384+
--hash=sha256:30be2844d0c939161a11073bfbaf645f1c7cb43f62f46cc6e4df1c119fb2a798 \
385+
--hash=sha256:36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a \
386+
--hash=sha256:56ad2a760b7a7882725a1eebf5657abbb3b5144eb26bcb47b52059357463c548 \
387+
--hash=sha256:5fc1d4d7ed265ef853579caf272686d1ed87cebdcd04f2a498f800ffc53dab71 \
388+
--hash=sha256:72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d \
389+
--hash=sha256:a38de2803ee6050309aac032676536c3d3b6a9804248537e38e098d0e14817ec \
390+
--hash=sha256:d36a8ef100f5bff3e9c3cea934b9e0d7ea277cb8210c7152d34a9a6c5830eadd \
391+
--hash=sha256:ddddbd8b066e743934a4200b3d54267a46db02106876d21cf31f7da7a96f98ea \
392+
--hash=sha256:fa27b048d32198cda6e9cff0bf768e8683d98743903b7e5d2b1f5098ded1d343
393+
# via -r requirements.in
394+
torch==2.4.1+cpu ; platform_machine == 'x86_64' \
395+
--hash=sha256:0c0a7cc4f7c74ff024d5a5e21230a01289b65346b27a626f6c815d94b4b8c955 \
396+
--hash=sha256:1dd062d296fb78aa7cfab8690bf03704995a821b5ef69cfc807af5c0831b4202 \
397+
--hash=sha256:2b03e20f37557d211d14e3fb3f71709325336402db132a1e0dd8b47392185baf \
398+
--hash=sha256:330e780f478707478f797fdc82c2a96e9b8c5f60b6f1f57bb6ad1dd5b1e7e97e \
399+
--hash=sha256:3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97 \
400+
--hash=sha256:3c99506980a2fb4b634008ccb758f42dd82f93ae2830c1e41f64536e310bf562 \
401+
--hash=sha256:76a6fe7b10491b650c630bc9ae328df40f79a948296b41d3b087b29a8a63cbad \
402+
--hash=sha256:833490a28ac156762ed6adaa7c695879564fa2fd0dc51bcf3fdb2c7b47dc55e6 \
403+
--hash=sha256:8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364 \
404+
--hash=sha256:c4f2c3c026e876d4dad7629170ec14fff48c076d6c2ae0e354ab3fdc09024f00
405+
# via -r requirements.in
406+
""",
407+
}[x],
408+
),
409+
available_interpreters = {
410+
"python_3_12_host": "unit_test_interpreter_target",
411+
},
412+
evaluate_markers = lambda _, requirements, **__: {
413+
# todo once 2692 is merged, this is going to be easier to test.
414+
key: [
415+
platform
416+
for platform in platforms
417+
if ("x86_64" in platform and "platform_machine ==" in key) or ("x86_64" not in platform and "platform_machine !=" in key)
418+
]
419+
for key, platforms in requirements.items()
420+
},
421+
simpleapi_download = mocksimpleapi_download,
422+
)
423+
424+
pypi.is_reproducible().equals(False)
425+
pypi.exposed_packages().contains_exactly({"pypi": ["torch"]})
426+
pypi.hub_group_map().contains_exactly({"pypi": {}})
427+
pypi.hub_whl_map().contains_exactly({"pypi": {
428+
"torch": {
429+
"pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": [
430+
struct(
431+
config_setting = None,
432+
filename = "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
433+
target_platforms = None,
434+
version = "3.12",
435+
),
436+
],
437+
"pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": [
438+
struct(
439+
config_setting = None,
440+
filename = "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
441+
target_platforms = None,
442+
version = "3.12",
443+
),
444+
],
445+
"pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": [
446+
struct(
447+
config_setting = None,
448+
filename = "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
449+
target_platforms = None,
450+
version = "3.12",
451+
),
452+
],
453+
"pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": [
454+
struct(
455+
config_setting = None,
456+
filename = "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
457+
target_platforms = None,
458+
version = "3.12",
459+
),
460+
],
461+
},
462+
}})
463+
pypi.whl_libraries().contains_exactly({
464+
"pypi_312_torch_cp312_cp312_linux_x86_64_8800deef": {
465+
"dep_template": "@pypi//{name}:{target}",
466+
"experimental_target_platforms": ["cp312_linux_x86_64"],
467+
"filename": "torch-2.4.1+cpu-cp312-cp312-linux_x86_64.whl",
468+
"python_interpreter_target": "unit_test_interpreter_target",
469+
"repo": "pypi_312",
470+
"requirement": "torch==2.4.1+cpu",
471+
"sha256": "8800deef0026011d502c0c256cc4b67d002347f63c3a38cd8e45f1f445c61364",
472+
"urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-linux_x86_64.whl"],
473+
},
474+
"pypi_312_torch_cp312_cp312_manylinux_2_17_aarch64_36109432": {
475+
"dep_template": "@pypi//{name}:{target}",
476+
"experimental_target_platforms": ["cp312_linux_aarch64"],
477+
"filename": "torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl",
478+
"python_interpreter_target": "unit_test_interpreter_target",
479+
"repo": "pypi_312",
480+
"requirement": "torch==2.4.1",
481+
"sha256": "36109432b10bd7163c9b30ce896f3c2cca1b86b9765f956a1594f0ff43091e2a",
482+
"urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl"],
483+
},
484+
"pypi_312_torch_cp312_cp312_win_amd64_3a570e5c": {
485+
"dep_template": "@pypi//{name}:{target}",
486+
"experimental_target_platforms": ["cp312_windows_x86_64"],
487+
"filename": "torch-2.4.1+cpu-cp312-cp312-win_amd64.whl",
488+
"python_interpreter_target": "unit_test_interpreter_target",
489+
"repo": "pypi_312",
490+
"requirement": "torch==2.4.1+cpu",
491+
"sha256": "3a570e5c553415cdbddfe679207327b3a3806b21c6adea14fba77684d1619e97",
492+
"urls": ["https://torch.index/whl/cpu/torch-2.4.1%2Bcpu-cp312-cp312-win_amd64.whl"],
493+
},
494+
"pypi_312_torch_cp312_none_macosx_11_0_arm64_72b484d5": {
495+
"dep_template": "@pypi//{name}:{target}",
496+
"experimental_target_platforms": ["cp312_osx_aarch64"],
497+
"filename": "torch-2.4.1-cp312-none-macosx_11_0_arm64.whl",
498+
"python_interpreter_target": "unit_test_interpreter_target",
499+
"repo": "pypi_312",
500+
"requirement": "torch==2.4.1",
501+
"sha256": "72b484d5b6cec1a735bf3fa5a1c4883d01748698c5e9cfdbeb4ffab7c7987e0d",
502+
"urls": ["https://torch.index/whl/cpu/torch-2.4.1-cp312-none-macosx_11_0_arm64.whl"],
503+
},
504+
})
505+
pypi.whl_mods().contains_exactly({})
506+
507+
_tests.append(_test_torch_experimental_index_url)
508+
335509
def _test_download_only_multiple(env):
336510
pypi = _parse_modules(
337511
env,

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy