Skip to content

Commit c99aaec

Browse files
aignasaignas
authored
feat: add a tool to update internal dependencies (#1321)
Before this change the updates to the dependencies would happen very seldomly, with this script, I propose we do it before each minor version release. Adding a shell script and adding a reminder to the release process may help with that.
1 parent 608ddb7 commit c99aaec

File tree

13 files changed

+595
-77
lines changed

13 files changed

+595
-77
lines changed

‎DEVELOPING.md

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,16 @@
11
# For Developers
22

3+
## Updating internal dependencies
4+
5+
1. Modify the `./python/pip_install/tools/requirements.txt` file and run:
6+
```
7+
bazel run //tools/private/update_deps:update_pip_deps
8+
```
9+
1. Bump the coverage dependencies using the script using:
10+
```
11+
bazel run //tools/private/update_deps:update_coverage_deps <VERSION>
12+
```
13+
314
## Releasing
415

516
Start from a clean checkout at `main`.

‎MODULE.bazel

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ internal_deps = use_extension("@rules_python//python/extensions/private:internal
1515
internal_deps.install()
1616
use_repo(
1717
internal_deps,
18+
# START: maintained by 'bazel run //tools/private:update_pip_deps'
1819
"pypi__build",
1920
"pypi__click",
2021
"pypi__colorama",
@@ -29,6 +30,7 @@ use_repo(
2930
"pypi__tomli",
3031
"pypi__wheel",
3132
"pypi__zipp",
33+
# END: maintained by 'bazel run //tools/private:update_pip_deps'
3234
)
3335

3436
# We need to do another use_extension call to expose the "pythons_hub"

‎python/pip_install/BUILD.bazel

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,18 @@ filegroup(
99
visibility = ["//:__pkg__"],
1010
)
1111

12+
filegroup(
13+
name = "repositories",
14+
srcs = ["repositories.bzl"],
15+
visibility = ["//tools/private/update_deps:__pkg__"],
16+
)
17+
18+
filegroup(
19+
name = "requirements_txt",
20+
srcs = ["tools/requirements.txt"],
21+
visibility = ["//tools/private/update_deps:__pkg__"],
22+
)
23+
1224
filegroup(
1325
name = "bzl",
1426
srcs = glob(["*.bzl"]) + [

‎python/pip_install/repositories.bzl

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
2020
load("//:version.bzl", "MINIMUM_BAZEL_VERSION")
2121

2222
_RULE_DEPS = [
23+
# START: maintained by 'bazel run //tools/private:update_pip_deps'
2324
(
2425
"pypi__build",
2526
"https://files.pythonhosted.org/packages/03/97/f58c723ff036a8d8b4d3115377c0a37ed05c1f68dd9a0d66dab5e82c5c1c/build-0.9.0-py3-none-any.whl",
@@ -35,11 +36,21 @@ _RULE_DEPS = [
3536
"https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl",
3637
"4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6",
3738
),
39+
(
40+
"pypi__importlib_metadata",
41+
"https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
42+
"bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
43+
),
3844
(
3945
"pypi__installer",
4046
"https://files.pythonhosted.org/packages/e5/ca/1172b6638d52f2d6caa2dd262ec4c811ba59eee96d54a7701930726bce18/installer-0.7.0-py3-none-any.whl",
4147
"05d1933f0a5ba7d8d6296bb6d5018e7c94fa473ceb10cf198a92ccea19c27b53",
4248
),
49+
(
50+
"pypi__more_itertools",
51+
"https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl",
52+
"c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb",
53+
),
4354
(
4455
"pypi__packaging",
4556
"https://files.pythonhosted.org/packages/8f/7b/42582927d281d7cb035609cd3a543ffac89b74f3f4ee8e1c50914bcb57eb/packaging-22.0-py3-none-any.whl",
@@ -75,21 +86,12 @@ _RULE_DEPS = [
7586
"https://files.pythonhosted.org/packages/bd/7c/d38a0b30ce22fc26ed7dbc087c6d00851fb3395e9d0dac40bec1f905030c/wheel-0.38.4-py3-none-any.whl",
7687
"b60533f3f5d530e971d6737ca6d58681ee434818fab630c83a734bb10c083ce8",
7788
),
78-
(
79-
"pypi__importlib_metadata",
80-
"https://files.pythonhosted.org/packages/d7/31/74dcb59a601b95fce3b0334e8fc9db758f78e43075f22aeb3677dfb19f4c/importlib_metadata-1.4.0-py2.py3-none-any.whl",
81-
"bdd9b7c397c273bcc9a11d6629a38487cd07154fa255a467bf704cd2c258e359",
82-
),
8389
(
8490
"pypi__zipp",
8591
"https://files.pythonhosted.org/packages/f4/50/cc72c5bcd48f6e98219fc4a88a5227e9e28b81637a99c49feba1d51f4d50/zipp-1.0.0-py2.py3-none-any.whl",
8692
"8dda78f06bd1674bd8720df8a50bb47b6e1233c503a4eed8e7810686bde37656",
8793
),
88-
(
89-
"pypi__more_itertools",
90-
"https://files.pythonhosted.org/packages/bd/3f/c4b3dbd315e248f84c388bd4a72b131a29f123ecacc37ffb2b3834546e42/more_itertools-8.13.0-py3-none-any.whl",
91-
"c5122bffc5f104d37c1626b8615b511f3427aa5389b94d61e5ef8236bfbc3ddb",
92-
),
94+
# END: maintained by 'bazel run //tools/private:update_pip_deps'
9395
]
9496

9597
_GENERIC_WHEEL = """\

‎python/pip_install/tools/requirements.txt

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
build==0.9
2+
click==8.0.1
3+
colorama
4+
importlib_metadata==1.4.0
5+
installer
6+
more_itertools==8.13.0
7+
packaging==22.0
8+
pep517
9+
pip==22.3.1
10+
pip_tools==6.12.1
11+
setuptools==60.10
12+
tomli
13+
wheel==0.38.4
14+
zipp==1.0.0

‎python/private/BUILD.bazel

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,12 @@ filegroup(
2424
visibility = ["//python:__pkg__"],
2525
)
2626

27+
filegroup(
28+
name = "coverage_deps",
29+
srcs = ["coverage_deps.bzl"],
30+
visibility = ["//tools/private/update_deps:__pkg__"],
31+
)
32+
2733
# Filegroup of bzl files that can be used by downstream rules for documentation generation
2834
filegroup(
2935
name = "bzl",

‎python/private/coverage_deps.bzl

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,7 @@ load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
1919
load("@bazel_tools//tools/build_defs/repo:utils.bzl", "maybe")
2020
load("//python/private:version_label.bzl", "version_label")
2121

22-
# Update with './tools/update_coverage_deps.py <version>'
23-
#START: managed by update_coverage_deps.py script
22+
# START: maintained by 'bazel run //tools/private:update_coverage_deps'
2423
_coverage_deps = {
2524
"cp310": {
2625
"aarch64-apple-darwin": (
@@ -95,7 +94,7 @@ _coverage_deps = {
9594
),
9695
},
9796
}
98-
#END: managed by update_coverage_deps.py script
97+
# END: maintained by 'bazel run //tools/private:update_coverage_deps'
9998

10099
_coverage_patch = Label("//python/private:coverage.patch")
101100

‎tools/private/update_deps/BUILD.bazel

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Copyright 2017 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
load("//python:py_binary.bzl", "py_binary")
15+
load("//python:py_library.bzl", "py_library")
16+
load("//python:py_test.bzl", "py_test")
17+
18+
licenses(["notice"])
19+
20+
py_library(
21+
name = "args",
22+
srcs = ["args.py"],
23+
imports = ["../../.."],
24+
deps = ["//python/runfiles"],
25+
)
26+
27+
py_library(
28+
name = "update_file",
29+
srcs = ["update_file.py"],
30+
imports = ["../../.."],
31+
)
32+
33+
py_binary(
34+
name = "update_coverage_deps",
35+
srcs = ["update_coverage_deps.py"],
36+
data = [
37+
"//python/private:coverage_deps",
38+
],
39+
env = {
40+
"UPDATE_FILE": "$(rlocationpath //python/private:coverage_deps)",
41+
},
42+
imports = ["../../.."],
43+
deps = [
44+
":args",
45+
":update_file",
46+
],
47+
)
48+
49+
py_binary(
50+
name = "update_pip_deps",
51+
srcs = ["update_pip_deps.py"],
52+
data = [
53+
"//:MODULE.bazel",
54+
"//python/pip_install:repositories",
55+
"//python/pip_install:requirements_txt",
56+
],
57+
env = {
58+
"MODULE_BAZEL": "$(rlocationpath //:MODULE.bazel)",
59+
"REPOSITORIES_BZL": "$(rlocationpath //python/pip_install:repositories)",
60+
"REQUIREMENTS_TXT": "$(rlocationpath //python/pip_install:requirements_txt)",
61+
},
62+
imports = ["../../.."],
63+
deps = [
64+
":args",
65+
":update_file",
66+
],
67+
)
68+
69+
py_test(
70+
name = "update_file_test",
71+
srcs = ["update_file_test.py"],
72+
imports = ["../../.."],
73+
deps = [
74+
":update_file",
75+
],
76+
)

‎tools/private/update_deps/args.py

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Copyright 2023 The Bazel Authors. All rights reserved.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
"""A small library for common arguments when updating files."""
16+
17+
import pathlib
18+
19+
from python.runfiles import runfiles
20+
21+
22+
def path_from_runfiles(input: str) -> pathlib.Path:
23+
"""A helper to create a path from runfiles.
24+
25+
Args:
26+
input: the string input to construct a path.
27+
28+
Returns:
29+
the pathlib.Path path to a file which is verified to exist.
30+
"""
31+
path = pathlib.Path(runfiles.Create().Rlocation(input))
32+
if not path.exists():
33+
raise ValueError(f"Path '{path}' does not exist")
34+
35+
return path

‎tools/update_coverage_deps.py renamed to ‎tools/private/update_deps/update_coverage_deps.py

Lines changed: 14 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import argparse
2323
import difflib
2424
import json
25+
import os
2526
import pathlib
2627
import sys
2728
import textwrap
@@ -30,6 +31,9 @@
3031
from typing import Any
3132
from urllib import request
3233

34+
from tools.private.update_deps.args import path_from_runfiles
35+
from tools.private.update_deps.update_file import update_file
36+
3337
# This should be kept in sync with //python:versions.bzl
3438
_supported_platforms = {
3539
# Windows is unsupported right now
@@ -110,64 +114,6 @@ def _map(
110114
)
111115

112116

113-
def _writelines(path: pathlib.Path, lines: list[str]):
114-
with open(path, "w") as f:
115-
f.writelines(lines)
116-
117-
118-
def _difflines(path: pathlib.Path, lines: list[str]):
119-
with open(path) as f:
120-
input = f.readlines()
121-
122-
rules_python = pathlib.Path(__file__).parent.parent
123-
p = path.relative_to(rules_python)
124-
125-
print(f"Diff of the changes that would be made to '{p}':")
126-
for line in difflib.unified_diff(
127-
input,
128-
lines,
129-
fromfile=f"a/{p}",
130-
tofile=f"b/{p}",
131-
):
132-
print(line, end="")
133-
134-
# Add an empty line at the end of the diff
135-
print()
136-
137-
138-
def _update_file(
139-
path: pathlib.Path,
140-
snippet: str,
141-
start_marker: str,
142-
end_marker: str,
143-
dry_run: bool = True,
144-
):
145-
with open(path) as f:
146-
input = f.readlines()
147-
148-
out = []
149-
skip = False
150-
for line in input:
151-
if skip:
152-
if not line.startswith(end_marker):
153-
continue
154-
155-
skip = False
156-
157-
out.append(line)
158-
159-
if not line.startswith(start_marker):
160-
continue
161-
162-
skip = True
163-
out.extend([f"{line}\n" for line in snippet.splitlines()])
164-
165-
if dry_run:
166-
_difflines(path, out)
167-
else:
168-
_writelines(path, out)
169-
170-
171117
def _parse_args() -> argparse.Namespace:
172118
parser = argparse.ArgumentParser(__doc__)
173119
parser.add_argument(
@@ -193,6 +139,12 @@ def _parse_args() -> argparse.Namespace:
193139
action="store_true",
194140
help="Wether to write to files",
195141
)
142+
parser.add_argument(
143+
"--update-file",
144+
type=path_from_runfiles,
145+
default=os.environ.get("UPDATE_FILE"),
146+
help="The path for the file to be updated, defaults to the value taken from UPDATE_FILE",
147+
)
196148
return parser.parse_args()
197149

198150

@@ -230,14 +182,12 @@ def main():
230182

231183
urls.sort(key=lambda x: f"{x.python}_{x.platform}")
232184

233-
rules_python = pathlib.Path(__file__).parent.parent
234-
235185
# Update the coverage_deps, which are used to register deps
236-
_update_file(
237-
path=rules_python / "python" / "private" / "coverage_deps.bzl",
186+
update_file(
187+
path=args.update_file,
238188
snippet=f"_coverage_deps = {repr(Deps(urls))}\n",
239-
start_marker="#START: managed by update_coverage_deps.py script",
240-
end_marker="#END: managed by update_coverage_deps.py script",
189+
start_marker="# START: maintained by 'bazel run //tools/private:update_coverage_deps'",
190+
end_marker="# END: maintained by 'bazel run //tools/private:update_coverage_deps'",
241191
dry_run=args.dry_run,
242192
)
243193

0 commit comments

Comments
 (0)
pFad - Phonifier reborn

Pfad - The Proxy pFad of © 2024 Garber Painting. All rights reserved.

Note: This service is not intended for secure transactions such as banking, social media, email, or purchasing. Use at your own risk. We assume no liability whatsoever for broken pages.


Alternative Proxies:

Alternative Proxy

pFad Proxy

pFad v3 Proxy

pFad v4 Proxy